Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Add turbopuffer embedder by default [LET-4253] (#4476)
Matthew Zhou516f2963e0f7
diff --git a/letta/helpers/tpuf_client.py b/letta/helpers/tpuf_client.pyindex afdebec54..e7e2c8b0b 100644--- a/letta/helpers/tpuf_client.py+++ b/letta/helpers/tpuf_client.py@@ -16,12 +16,12 @@ logger = logging.getLogger(__name__)def should_use_tpuf() -> bool:# We need OpenAI since we default to their embedding model- return bool(settings.use_tpuf) and bool(settings.tpuf_api_key)+ return bool(settings.use_tpuf) and bool(settings.tpuf_api_key) and bool(model_settings.openai_api_key)def should_use_tpuf_for_messages() -> bool:"""Check if Turbopuffer should be used for messages."""- return should_use_tpuf() and bool(settings.embed_all_messages) and bool(model_settings.openai_api_key)+ return should_use_tpuf() and bool(settings.embed_all_messages)class TurbopufferClient:@@ -1113,3 +1113,309 @@ class TurbopufferClient:except Exception as e:logger.error(f"Failed to delete all messages from Turbopuffer: {e}")raise++ # file/source passage methods++ @trace_method+ async def _get_file_passages_namespace_name(self, organization_id: str) -> str:+ """Get namespace name for file passages (org-scoped).++ Args:+ organization_id: Organization ID for namespace generation++ Returns:+ The org-scoped namespace name for file passages+ """+ environment = settings.environment+ if environment:+ namespace_name = f"file_passages_{organization_id}_{environment.lower()}"+ else:+ namespace_name = f"file_passages_{organization_id}"++ return namespace_name++ @trace_method+ async def insert_file_passages(+ self,+ source_id: str,+ file_id: str,+ text_chunks: List[str],+ organization_id: str,+ actor: "PydanticUser",+ created_at: Optional[datetime] = None,+ ) -> List[PydanticPassage]:+ """Insert file passages into Turbopuffer using org-scoped namespace.++ Args:+ source_id: ID of the source containing the file+ file_id: ID of the file+ text_chunks: List of text chunks to store+ organization_id: Organization ID for the passages+ actor: User actor for embedding generation+ created_at: Optional timestamp for retroactive entries (defaults to current UTC time)++ Returns:+ List of PydanticPassage objects that were inserted+ """+ from turbopuffer import AsyncTurbopuffer++ if not text_chunks:+ return []++ # generate embeddings using the default config+ embeddings = await self._generate_embeddings(text_chunks, actor)++ namespace_name = await self._get_file_passages_namespace_name(organization_id)++ # handle timestamp - ensure UTC+ if created_at is None:+ timestamp = datetime.now(timezone.utc)+ else:+ # ensure the provided timestamp is timezone-aware and in UTC+ if created_at.tzinfo is None:+ # assume UTC if no timezone provided+ timestamp = created_at.replace(tzinfo=timezone.utc)+ else:+ # convert to UTC if in different timezone+ timestamp = created_at.astimezone(timezone.utc)++ # prepare column-based data for turbopuffer - optimized for batch insert+ ids = []+ vectors = []+ texts = []+ organization_ids = []+ source_ids = []+ file_ids = []+ created_ats = []+ passages = []++ for idx, (text, embedding) in enumerate(zip(text_chunks, embeddings)):+ passage = PydanticPassage(+ text=text,+ file_id=file_id,+ source_id=source_id,+ embedding=embedding,+ embedding_config=self.default_embedding_config,+ organization_id=actor.organization_id,+ )+ passages.append(passage)++ # append to columns+ ids.append(passage.id)+ vectors.append(embedding)+ texts.append(text)+ organization_ids.append(organization_id)+ source_ids.append(source_id)+ file_ids.append(file_id)+ created_ats.append(timestamp)++ # build column-based upsert data+ upsert_columns = {+ "id": ids,+ "vector": vectors,+ "text": texts,+ "organization_id": organization_ids,+ "source_id": source_ids,+ "file_id": file_ids,+ "created_at": created_ats,+ }++ try:+ # use AsyncTurbopuffer as a context manager for proper resource cleanup+ async with AsyncTurbopuffer(api_key=self.api_key, region=self.region) as client:+ namespace = client.namespace(namespace_name)+ # turbopuffer recommends column-based writes for performance+ await namespace.write(+ upsert_columns=upsert_columns,+ distance_metric="cosine_distance",+ schema={"text": {"type": "string", "full_text_search": True}},+ )+ logger.info(f"Successfully inserted {len(ids)} file passages to Turbopuffer for source {source_id}, file {file_id}")+ return passages++ except Exception as e:+ logger.error(f"Failed to insert file passages to Turbopuffer: {e}")+ # check if it's a duplicate ID error+ if "duplicate" in str(e).lower():+ logger.error("Duplicate passage IDs detected in batch")+ raise++ @trace_method+ async def query_file_passages(+ self,+ source_ids: List[str],+ organization_id: str,+ actor: "PydanticUser",+ query_text: Optional[str] = None,+ search_mode: str = "vector", # "vector", "fts", "hybrid"+ top_k: int = 10,+ file_id: Optional[str] = None, # optional filter by specific file+ vector_weight: float = 0.5,+ fts_weight: float = 0.5,+ ) -> List[Tuple[PydanticPassage, float, dict]]:+ """Query file passages from Turbopuffer using org-scoped namespace.++ Args:+ source_ids: List of source IDs to query+ organization_id: Organization ID for namespace lookup+ actor: User actor for embedding generation+ query_text: Text query for search+ search_mode: Search mode - "vector", "fts", or "hybrid" (default: "vector")+ top_k: Number of results to return+ file_id: Optional file ID to filter results to a specific file+ vector_weight: Weight for vector search results in hybrid mode (default: 0.5)+ fts_weight: Weight for FTS results in hybrid mode (default: 0.5)++ Returns:+ List of (passage, score, metadata) tuples with relevance rankings+ """+ # generate embedding for vector/hybrid search if query_text is provided+ query_embedding = None+ if query_text and search_mode in ["vector", "hybrid"]:+ embeddings = await self._generate_embeddings([query_text], actor)+ query_embedding = embeddings[0]++ # check if we should fallback to timestamp-based retrieval+ if query_embedding is None and query_text is None and search_mode not in ["timestamp"]:+ # fallback to retrieving most recent passages when no search query is provided+ search_mode = "timestamp"++ namespace_name = await self._get_file_passages_namespace_name(organization_id)++ # build filters - always filter by source_ids+ if len(source_ids) == 1:+ # single source_id, use Eq for efficiency+ filters = [("source_id", "Eq", source_ids[0])]+ else:+ # multiple source_ids, use In operator+ filters = [("source_id", "In", source_ids)]++ # add file filter if specified+ if file_id:+ filters.append(("file_id", "Eq", file_id))++ # combine filters+ final_filter = filters[0] if len(filters) == 1 else ("And", filters)++ try:+ # use generic query executor+ result = await self._execute_query(+ namespace_name=namespace_name,+ search_mode=search_mode,+ query_embedding=query_embedding,+ query_text=query_text,+ top_k=top_k,+ include_attributes=["text", "organization_id", "source_id", "file_id", "created_at"],+ filters=final_filter,+ vector_weight=vector_weight,+ fts_weight=fts_weight,+ )++ # process results based on search mode+ if search_mode == "hybrid":+ # for hybrid mode, we get a multi-query response+ vector_results = self._process_file_query_results(result.results[0])+ fts_results = self._process_file_query_results(result.results[1], is_fts=True)+ # use RRF and include metadata with ranks+ results_with_metadata = self._reciprocal_rank_fusion(+ vector_results=[passage for passage, _ in vector_results],+ fts_results=[passage for passage, _ in fts_results],+ get_id_func=lambda p: p.id,+ vector_weight=vector_weight,+ fts_weight=fts_weight,+ top_k=top_k,+ )+ return results_with_metadata+ else:+ # for single queries (vector, fts, timestamp) - add basic metadata+ is_fts = search_mode == "fts"+ results = self._process_file_query_results(result, is_fts=is_fts)+ # add simple metadata for single search modes+ results_with_metadata = []+ for idx, (passage, score) in enumerate(results):+ metadata = {+ "combined_score": score,+ f"{search_mode}_rank": idx + 1, # add the rank for this search mode+ }+ results_with_metadata.append((passage, score, metadata))+ return results_with_metadata++ except Exception as e:+ logger.error(f"Failed to query file passages from Turbopuffer: {e}")+ raise++ def _process_file_query_results(self, result, is_fts: bool = False) -> List[Tuple[PydanticPassage, float]]:+ """Process results from a file query into passage objects with scores."""+ passages_with_scores = []++ for row in result.rows:+ # build metadata+ metadata = {}++ # create a passage with minimal fields - embeddings are not returned from Turbopuffer+ passage = PydanticPassage(+ id=row.id,+ text=getattr(row, "text", ""),+ organization_id=getattr(row, "organization_id", None),+ source_id=getattr(row, "source_id", None), # get source_id from the row+ file_id=getattr(row, "file_id", None),+ created_at=getattr(row, "created_at", None),+ metadata_=metadata,+ tags=[],+ # set required fields to empty/default values since we don't store embeddings+ embedding=[], # empty embedding since we don't return it from Turbopuffer+ embedding_config=self.default_embedding_config,+ )++ # handle score based on search type+ if is_fts:+ # for FTS, use the BM25 score directly (higher is better)+ score = getattr(row, "$score", 0.0)+ else:+ # for vector search, convert distance to similarity score+ distance = getattr(row, "$dist", 0.0)+ score = 1.0 - distance++ passages_with_scores.append((passage, score))++ return passages_with_scores++ @trace_method+ async def delete_file_passages(self, source_id: str, file_id: str, organization_id: str) -> bool:+ """Delete all passages for a specific file from Turbopuffer."""+ from turbopuffer import AsyncTurbopuffer++ namespace_name = await self._get_file_passages_namespace_name(organization_id)++ try:+ async with AsyncTurbopuffer(api_key=self.api_key, region=self.region) as client:+ namespace = client.namespace(namespace_name)+ # use delete_by_filter to only delete passages for this file+ # need to filter by both source_id and file_id+ filter_expr = ("And", [("source_id", "Eq", source_id), ("file_id", "Eq", file_id)])+ result = await namespace.write(delete_by_filter=filter_expr)+ logger.info(+ f"Successfully deleted passages for file {file_id} from source {source_id} (deleted {result.rows_affected} rows)"+ )+ return True+ except Exception as e:+ logger.error(f"Failed to delete file passages from Turbopuffer: {e}")+ raise++ @trace_method+ async def delete_source_passages(self, source_id: str, organization_id: str) -> bool:+ """Delete all passages for a source from Turbopuffer."""+ from turbopuffer import AsyncTurbopuffer++ namespace_name = await self._get_file_passages_namespace_name(organization_id)++ try:+ async with AsyncTurbopuffer(api_key=self.api_key, region=self.region) as client:+ namespace = client.namespace(namespace_name)+ # delete all passages for this source+ result = await namespace.write(delete_by_filter=("source_id", "Eq", source_id))+ logger.info(f"Successfully deleted all passages for source {source_id} (deleted {result.rows_affected} rows)")+ return True+ except Exception as e:+ logger.error(f"Failed to delete source passages from Turbopuffer: {e}")+ raisediff --git a/letta/server/rest_api/routers/v1/folders.py b/letta/server/rest_api/routers/v1/folders.pyindex dcf984746..84a597231 100644--- a/letta/server/rest_api/routers/v1/folders.py+++ b/letta/server/rest_api/routers/v1/folders.py@@ -15,6 +15,7 @@ from letta.helpers.pinecone_utils import (delete_source_records_from_pinecone_index,should_use_pinecone,)+from letta.helpers.tpuf_client import should_use_tpuffrom letta.log import get_loggerfrom letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentState@@ -191,7 +192,13 @@ async def delete_folder(files = await server.file_manager.list_files(folder_id, actor)file_ids = [f.id for f in files]- if should_use_pinecone():+ if should_use_tpuf():+ logger.info(f"Deleting folder {folder_id} from Turbopuffer")+ from letta.helpers.tpuf_client import TurbopufferClient++ tpuf_client = TurbopufferClient()+ await tpuf_client.delete_source_passages(source_id=folder_id, organization_id=actor.organization_id)+ elif should_use_pinecone():logger.info(f"Deleting folder {folder_id} from pinecone index")await delete_source_records_from_pinecone_index(source_id=folder_id, actor=actor)@@ -450,7 +457,13 @@ async def delete_file_from_folder(await server.remove_file_from_context_windows(source_id=folder_id, file_id=deleted_file.id, actor=actor)- if should_use_pinecone():+ if should_use_tpuf():+ logger.info(f"Deleting file {file_id} from Turbopuffer")+ from letta.helpers.tpuf_client import TurbopufferClient++ tpuf_client = TurbopufferClient()+ await tpuf_client.delete_file_passages(source_id=folder_id, file_id=file_id, organization_id=actor.organization_id)+ elif should_use_pinecone():logger.info(f"Deleting file {file_id} from pinecone index")await delete_file_records_from_pinecone_index(file_id=file_id, actor=actor)@@ -496,10 +509,15 @@ async def load_file_to_source_cloud(else:file_parser = MarkitdownFileParser()- using_pinecone = should_use_pinecone()- if using_pinecone:+ # determine which embedder to use - turbopuffer takes precedence+ if should_use_tpuf():+ from letta.services.file_processor.embedder.turbopuffer_embedder import TurbopufferEmbedder++ embedder = TurbopufferEmbedder(embedding_config=embedding_config)+ elif should_use_pinecone():embedder = PineconeEmbedder(embedding_config=embedding_config)else:embedder = OpenAIEmbedder(embedding_config=embedding_config)- file_processor = FileProcessor(file_parser=file_parser, embedder=embedder, actor=actor, using_pinecone=using_pinecone)++ file_processor = FileProcessor(file_parser=file_parser, embedder=embedder, actor=actor)await file_processor.process(agent_states=agent_states, source_id=source_id, content=content, file_metadata=file_metadata)diff --git a/letta/server/rest_api/routers/v1/sources.py b/letta/server/rest_api/routers/v1/sources.pyindex c9d554072..a5fee7b80 100644--- a/letta/server/rest_api/routers/v1/sources.py+++ b/letta/server/rest_api/routers/v1/sources.py@@ -15,6 +15,7 @@ from letta.helpers.pinecone_utils import (delete_source_records_from_pinecone_index,should_use_pinecone,)+from letta.helpers.tpuf_client import should_use_tpuffrom letta.log import get_loggerfrom letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentState@@ -189,7 +190,13 @@ async def delete_source(files = await server.file_manager.list_files(source_id, actor)file_ids = [f.id for f in files]- if should_use_pinecone():+ if should_use_tpuf():+ logger.info(f"Deleting source {source_id} from Turbopuffer")+ from letta.helpers.tpuf_client import TurbopufferClient++ tpuf_client = TurbopufferClient()+ await tpuf_client.delete_source_passages(source_id=source_id, organization_id=actor.organization_id)+ elif should_use_pinecone():logger.info(f"Deleting source {source_id} from pinecone index")await delete_source_records_from_pinecone_index(source_id=source_id, actor=actor)@@ -435,7 +442,13 @@ async def delete_file_from_source(await server.remove_file_from_context_windows(source_id=source_id, file_id=deleted_file.id, actor=actor)- if should_use_pinecone():+ if should_use_tpuf():+ logger.info(f"Deleting file {file_id} from Turbopuffer")+ from letta.helpers.tpuf_client import TurbopufferClient++ tpuf_client = TurbopufferClient()+ await tpuf_client.delete_file_passages(source_id=source_id, file_id=file_id, organization_id=actor.organization_id)+ elif should_use_pinecone():logger.info(f"Deleting file {file_id} from pinecone index")await delete_file_records_from_pinecone_index(file_id=file_id, actor=actor)@@ -481,10 +494,15 @@ async def load_file_to_source_cloud(else:file_parser = MarkitdownFileParser()- using_pinecone = should_use_pinecone()- if using_pinecone:+ # determine which embedder to use - turbopuffer takes precedence+ if should_use_tpuf():+ from letta.services.file_processor.embedder.turbopuffer_embedder import TurbopufferEmbedder++ embedder = TurbopufferEmbedder(embedding_config=embedding_config)+ elif should_use_pinecone():embedder = PineconeEmbedder(embedding_config=embedding_config)else:embedder = OpenAIEmbedder(embedding_config=embedding_config)- file_processor = FileProcessor(file_parser=file_parser, embedder=embedder, actor=actor, using_pinecone=using_pinecone)++ file_processor = FileProcessor(file_parser=file_parser, embedder=embedder, actor=actor)await file_processor.process(agent_states=agent_states, source_id=source_id, content=content, file_metadata=file_metadata)diff --git a/letta/services/agent_serialization_manager.py b/letta/services/agent_serialization_manager.pyindex bc0e07c3e..0cbabe4c6 100644--- a/letta/services/agent_serialization_manager.py+++ b/letta/services/agent_serialization_manager.py@@ -12,6 +12,7 @@ from letta.errors import (AgentNotFoundForExportError,)from letta.helpers.pinecone_utils import should_use_pinecone+from letta.helpers.tpuf_client import should_use_tpuffrom letta.log import get_loggerfrom letta.schemas.agent import AgentState, CreateAgentfrom letta.schemas.agent_file import (@@ -29,7 +30,7 @@ from letta.schemas.agent_file import ()from letta.schemas.block import Blockfrom letta.schemas.embedding_config import EmbeddingConfig-from letta.schemas.enums import FileProcessingStatus+from letta.schemas.enums import FileProcessingStatus, VectorDBProviderfrom letta.schemas.file import FileMetadatafrom letta.schemas.group import Group, GroupCreatefrom letta.schemas.mcp import MCPServer@@ -90,7 +91,6 @@ class AgentSerializationManager:self.file_agent_manager = file_agent_managerself.message_manager = message_managerself.file_parser = MistralFileParser() if settings.mistral_api_key else MarkitdownFileParser()- self.using_pinecone = should_use_pinecone()# ID mapping state for exportself._db_to_file_ids: Dict[str, str] = {}@@ -588,7 +588,12 @@ class AgentSerializationManager:if schema.files and any(f.content for f in schema.files):# Use override embedding config if provided, otherwise use agent's configembedder_config = override_embedding_config if override_embedding_config else schema.agents[0].embedding_config- if should_use_pinecone():+ # determine which embedder to use - turbopuffer takes precedence+ if should_use_tpuf():+ from letta.services.file_processor.embedder.turbopuffer_embedder import TurbopufferEmbedder++ embedder = TurbopufferEmbedder(embedding_config=embedder_config)+ elif should_use_pinecone():embedder = PineconeEmbedder(embedding_config=embedder_config)else:embedder = OpenAIEmbedder(embedding_config=embedder_config)@@ -596,7 +601,6 @@ class AgentSerializationManager:file_parser=self.file_parser,embedder=embedder,actor=actor,- using_pinecone=self.using_pinecone,)for file_schema in schema.files:diff --git a/letta/services/file_processor/embedder/base_embedder.py b/letta/services/file_processor/embedder/base_embedder.pyindex b9310c3e1..b2a6408b1 100644--- a/letta/services/file_processor/embedder/base_embedder.py+++ b/letta/services/file_processor/embedder/base_embedder.py@@ -2,6 +2,7 @@ from abc import ABC, abstractmethodfrom typing import Listfrom letta.log import get_logger+from letta.schemas.enums import VectorDBProviderfrom letta.schemas.passage import Passagefrom letta.schemas.user import User@@ -11,6 +12,10 @@ logger = get_logger(__name__)class BaseEmbedder(ABC):"""Abstract base class for embedding generation"""+ def __init__(self):+ # Default to NATIVE, subclasses will override this+ self.vector_db_type = VectorDBProvider.NATIVE+@abstractmethodasync def generate_embedded_passages(self, file_id: str, source_id: str, chunks: List[str], actor: User) -> List[Passage]:"""Generate embeddings for chunks with batching and concurrent processing"""diff --git a/letta/services/file_processor/embedder/openai_embedder.py b/letta/services/file_processor/embedder/openai_embedder.pyindex b55ba9366..77adbd852 100644--- a/letta/services/file_processor/embedder/openai_embedder.py+++ b/letta/services/file_processor/embedder/openai_embedder.py@@ -19,6 +19,10 @@ class OpenAIEmbedder(BaseEmbedder):"""OpenAI-based embedding generation"""def __init__(self, embedding_config: Optional[EmbeddingConfig] = None):+ super().__init__()+ # OpenAI embedder uses the native vector db (PostgreSQL)+ # self.vector_db_type already set to VectorDBProvider.NATIVE by parent+self.default_embedding_config = (EmbeddingConfig.default_config(model_name="text-embedding-3-small", provider="openai")if model_settings.openai_api_keydiff --git a/letta/services/file_processor/embedder/pinecone_embedder.py b/letta/services/file_processor/embedder/pinecone_embedder.pyindex c218807e0..f11aafed8 100644--- a/letta/services/file_processor/embedder/pinecone_embedder.py+++ b/letta/services/file_processor/embedder/pinecone_embedder.py@@ -4,6 +4,7 @@ from letta.helpers.pinecone_utils import upsert_file_records_to_pinecone_indexfrom letta.log import get_loggerfrom letta.otel.tracing import log_event, trace_methodfrom letta.schemas.embedding_config import EmbeddingConfig+from letta.schemas.enums import VectorDBProviderfrom letta.schemas.passage import Passagefrom letta.schemas.user import Userfrom letta.services.file_processor.embedder.base_embedder import BaseEmbedder@@ -20,6 +21,10 @@ class PineconeEmbedder(BaseEmbedder):"""Pinecone-based embedding generation"""def __init__(self, embedding_config: Optional[EmbeddingConfig] = None):+ super().__init__()+ # set the vector db type for pinecone+ self.vector_db_type = VectorDBProvider.PINECONE+if not PINECONE_AVAILABLE:raise ImportError("Pinecone package is not installed. Install it with: pip install pinecone")@@ -28,7 +33,6 @@ class PineconeEmbedder(BaseEmbedder):embedding_config = EmbeddingConfig.default_config(provider="pinecone")self.embedding_config = embedding_config- super().__init__()@trace_methodasync def generate_embedded_passages(self, file_id: str, source_id: str, chunks: List[str], actor: User) -> List[Passage]:diff --git a/letta/services/file_processor/embedder/turbopuffer_embedder.py b/letta/services/file_processor/embedder/turbopuffer_embedder.pynew file mode 100644index 000000000..c17b28c3b--- /dev/null+++ b/letta/services/file_processor/embedder/turbopuffer_embedder.py@@ -0,0 +1,71 @@+from typing import List, Optional++from letta.helpers.tpuf_client import TurbopufferClient+from letta.log import get_logger+from letta.otel.tracing import log_event, trace_method+from letta.schemas.embedding_config import EmbeddingConfig+from letta.schemas.enums import VectorDBProvider+from letta.schemas.passage import Passage+from letta.schemas.user import User+from letta.services.file_processor.embedder.base_embedder import BaseEmbedder++logger = get_logger(__name__)+++class TurbopufferEmbedder(BaseEmbedder):+ """Turbopuffer-based embedding generation and storage"""++ def __init__(self, embedding_config: Optional[EmbeddingConfig] = None):+ super().__init__()+ # set the vector db type for turbopuffer+ self.vector_db_type = VectorDBProvider.TPUF+ # use the default embedding config from TurbopufferClient if not provided+ self.embedding_config = embedding_config or TurbopufferClient.default_embedding_config+ self.tpuf_client = TurbopufferClient()++ @trace_method+ async def generate_embedded_passages(self, file_id: str, source_id: str, chunks: List[str], actor: User) -> List[Passage]:+ """Generate embeddings and store in Turbopuffer, then return Passage objects"""+ if not chunks:+ return []++ logger.info(f"Generating embeddings for {len(chunks)} chunks using Turbopuffer")+ log_event(+ "turbopuffer_embedder.generation_started",+ {+ "total_chunks": len(chunks),+ "file_id": file_id,+ "source_id": source_id,+ "embedding_model": self.embedding_config.embedding_model,+ },+ )++ try:+ # insert passages to Turbopuffer - it will handle embedding generation internally+ passages = await self.tpuf_client.insert_file_passages(+ source_id=source_id,+ file_id=file_id,+ text_chunks=chunks,+ organization_id=actor.organization_id,+ actor=actor,+ )++ logger.info(f"Successfully generated and stored {len(passages)} passages in Turbopuffer")+ log_event(+ "turbopuffer_embedder.generation_completed",+ {+ "passages_created": len(passages),+ "total_chunks_processed": len(chunks),+ "file_id": file_id,+ "source_id": source_id,+ },+ )+ return passages++ except Exception as e:+ logger.error(f"Failed to generate embeddings with Turbopuffer: {str(e)}")+ log_event(+ "turbopuffer_embedder.generation_failed",+ {"error": str(e), "error_type": type(e).__name__, "file_id": file_id, "source_id": source_id},+ )+ raisediff --git a/letta/services/file_processor/file_processor.py b/letta/services/file_processor/file_processor.pyindex 6e63a1ec4..529ea70d5 100644--- a/letta/services/file_processor/file_processor.py+++ b/letta/services/file_processor/file_processor.py@@ -6,7 +6,7 @@ from letta.log import get_loggerfrom letta.otel.context import get_ctx_attributesfrom letta.otel.tracing import log_event, trace_methodfrom letta.schemas.agent import AgentState-from letta.schemas.enums import FileProcessingStatus+from letta.schemas.enums import FileProcessingStatus, VectorDBProviderfrom letta.schemas.file import FileMetadatafrom letta.schemas.passage import Passagefrom letta.schemas.user import User@@ -30,7 +30,6 @@ class FileProcessor:file_parser: FileParser,embedder: BaseEmbedder,actor: User,- using_pinecone: bool,max_file_size: int = 50 * 1024 * 1024, # 50MB default):self.file_parser = file_parser@@ -42,7 +41,8 @@ class FileProcessor:self.job_manager = JobManager()self.agent_manager = AgentManager()self.actor = actor- self.using_pinecone = using_pinecone+ # get vector db type from the embedder+ self.vector_db_type = embedder.vector_db_typeasync def _chunk_and_embed_with_fallback(self, file_metadata: FileMetadata, ocr_response, source_id: str) -> List:"""Chunk text and generate embeddings with fallback to default chunker if needed"""@@ -218,7 +218,7 @@ class FileProcessor:source_id=source_id,)- if not self.using_pinecone:+ if self.vector_db_type == VectorDBProvider.NATIVE:all_passages = await self.passage_manager.create_many_source_passages_async(passages=all_passages,file_metadata=file_metadata,@@ -241,7 +241,8 @@ class FileProcessor:)# update job status- if not self.using_pinecone:+ # pinecone completes slowly, so gets updated later+ if self.vector_db_type != VectorDBProvider.PINECONE:await self.file_manager.update_file_status(file_id=file_metadata.id,actor=self.actor,@@ -317,14 +318,15 @@ class FileProcessor:)# Create passages in database (unless using Pinecone)- if not self.using_pinecone:+ if self.vector_db_type == VectorDBProvider.NATIVE:all_passages = await self.passage_manager.create_many_source_passages_async(passages=all_passages, file_metadata=file_metadata, actor=self.actor)log_event("file_processor.import_passages_created", {"filename": filename, "total_passages": len(all_passages)})# Update file status to completed (valid transition from EMBEDDING)- if not self.using_pinecone:+ # pinecone completes slowly, so gets updated later+ if self.vector_db_type != VectorDBProvider.PINECONE:await self.file_manager.update_file_status(file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.COMPLETED)diff --git a/letta/services/source_manager.py b/letta/services/source_manager.pyindex efe9e650a..8f10baebf 100644--- a/letta/services/source_manager.py+++ b/letta/services/source_manager.py@@ -3,6 +3,7 @@ from typing import List, Optional, Unionfrom sqlalchemy import and_, exists, select+from letta.helpers.pinecone_utils import should_use_pineconefrom letta.helpers.tpuf_client import should_use_tpuffrom letta.orm import Agent as AgentModelfrom letta.orm.errors import NoResultFound@@ -18,6 +19,18 @@ from letta.utils import enforce_types, printdclass SourceManager:+ def _get_vector_db_provider(self) -> VectorDBProvider:+ """+ determine which vector db provider to use based on configuration.+ turbopuffer takes precedence when available.+ """+ if should_use_tpuf():+ return VectorDBProvider.TPUF+ elif should_use_pinecone():+ return VectorDBProvider.PINECONE+ else:+ return VectorDBProvider.NATIVE+"""Manager class to handle business logic related to Sources."""@trace_method@@ -52,7 +65,7 @@ class SourceManager:if db_source:return db_sourceelse:- vector_db_provider = VectorDBProvider.TPUF if should_use_tpuf() else VectorDBProvider.NATIVE+ vector_db_provider = self._get_vector_db_provider()async with db_registry.async_session() as session:# Provide default embedding config if not given@@ -96,7 +109,7 @@ class SourceManager:Returns:List of created/updated sources"""- vector_db_provider = VectorDBProvider.TPUF if should_use_tpuf() else VectorDBProvider.NATIVE+ vector_db_provider = self._get_vector_db_provider()for pydantic_source in pydantic_sources:pydantic_source.vector_db_provider = vector_db_providerdiff --git a/letta/services/tool_executor/files_tool_executor.py b/letta/services/tool_executor/files_tool_executor.pyindex 43a3fd97b..251d0320b 100644--- a/letta/services/tool_executor/files_tool_executor.py+++ b/letta/services/tool_executor/files_tool_executor.py@@ -5,10 +5,13 @@ from typing import Any, Dict, List, Optionalfrom letta.constants import PINECONE_TEXT_FIELD_NAMEfrom letta.functions.types import FileOpenRequestfrom letta.helpers.pinecone_utils import search_pinecone_index, should_use_pinecone+from letta.helpers.tpuf_client import should_use_tpuffrom letta.log import get_loggerfrom letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentState+from letta.schemas.enums import VectorDBProviderfrom letta.schemas.sandbox_config import SandboxConfig+from letta.schemas.source import Sourcefrom letta.schemas.tool import Toolfrom letta.schemas.tool_execution_result import ToolExecutionResultfrom letta.schemas.user import User@@ -554,18 +557,140 @@ class LettaFileToolExecutor(ToolExecutor):self.logger.info(f"Semantic search started for agent {agent_state.id} with query '{query}' (limit: {limit})")- # Check if Pinecone is enabled and use it if available- if should_use_pinecone():- return await self._search_files_pinecone(agent_state, query, limit)- else:- return await self._search_files_traditional(agent_state, query, limit)+ # Check which vector DB to use - Turbopuffer takes precedence+ attached_sources = await self.agent_manager.list_attached_sources_async(agent_id=agent_state.id, actor=self.actor)+ attached_tpuf_sources = [source for source in attached_sources if source.vector_db_provider == VectorDBProvider.TPUF]+ attached_pinecone_sources = [source for source in attached_sources if source.vector_db_provider == VectorDBProvider.PINECONE]++ if not attached_tpuf_sources and not attached_pinecone_sources:+ return await self._search_files_native(agent_state, query, limit)++ results = []++ # If both have items, we half the limit roughly+ # TODO: This is very hacky bc it skips the re-ranking - but this is a temporary stopgap while we think about migrating data++ if attached_tpuf_sources and attached_pinecone_sources:+ limit = max(limit // 2, 1)++ if should_use_tpuf() and attached_tpuf_sources:+ tpuf_result = await self._search_files_turbopuffer(agent_state, attached_tpuf_sources, query, limit)+ results.append(tpuf_result)++ if should_use_pinecone() and attached_pinecone_sources:+ pinecone_result = await self._search_files_pinecone(agent_state, attached_pinecone_sources, query, limit)+ results.append(pinecone_result)++ # combine results from both sources+ if results:+ return "\n\n".join(results)++ # fallback if no results from either source+ return "No results found"++ async def _search_files_turbopuffer(self, agent_state: AgentState, attached_sources: List[Source], query: str, limit: int) -> str:+ """Search files using Turbopuffer vector database."""++ # Get attached sources+ source_ids = [source.id for source in attached_sources]+ if not source_ids:+ return "No valid source IDs found for attached files"++ # Get all attached files for this agent+ file_agents = await self.files_agents_manager.list_files_for_agent(+ agent_id=agent_state.id, per_file_view_window_char_limit=agent_state.per_file_view_window_char_limit, actor=self.actor+ )+ if not file_agents:+ return "No files are currently attached to search"++ # Create a map of file_id to file_name for quick lookup+ file_map = {fa.file_id: fa.file_name for fa in file_agents}++ results = []+ total_hits = 0+ files_with_matches = {}++ try:+ from letta.helpers.tpuf_client import TurbopufferClient++ tpuf_client = TurbopufferClient()++ # Query Turbopuffer for all sources at once+ search_results = await tpuf_client.query_file_passages(+ source_ids=source_ids, # pass all source_ids as a list+ organization_id=self.actor.organization_id,+ actor=self.actor,+ query_text=query,+ search_mode="hybrid", # use hybrid search for best results+ top_k=limit,+ )++ # Process search results+ for passage, score, metadata in search_results:+ if total_hits >= limit:+ break++ total_hits += 1++ # get file name from our map+ file_name = file_map.get(passage.file_id, "Unknown File")- async def _search_files_pinecone(self, agent_state: AgentState, query: str, limit: int) -> str:+ # group by file name+ if file_name not in files_with_matches:+ files_with_matches[file_name] = []+ files_with_matches[file_name].append({"text": passage.text, "score": score, "passage_id": passage.id})++ except Exception as e:+ self.logger.error(f"Turbopuffer search failed: {str(e)}")+ raise e++ if not files_with_matches:+ return f"No semantic matches found in Turbopuffer for query: '{query}'"++ # Format results+ passage_num = 0+ for file_name, matches in files_with_matches.items():+ for match in matches:+ passage_num += 1++ # format each passage with terminal-style header+ score_display = f"(score: {match['score']:.3f})"+ passage_header = f"\n=== {file_name} (passage #{passage_num}) {score_display} ==="++ # format the passage text+ passage_text = match["text"].strip()+ lines = passage_text.splitlines()+ formatted_lines = []+ for line in lines[:20]: # limit to first 20 lines per passage+ formatted_lines.append(f" {line}")++ if len(lines) > 20:+ formatted_lines.append(f" ... [truncated {len(lines) - 20} more lines]")++ passage_content = "\n".join(formatted_lines)+ results.append(f"{passage_header}\n{passage_content}")++ # mark access for files that had matches+ if files_with_matches:+ matched_file_names = [name for name in files_with_matches.keys() if name != "Unknown File"]+ if matched_file_names:+ await self.files_agents_manager.mark_access_bulk(agent_id=agent_state.id, file_names=matched_file_names, actor=self.actor)++ # create summary header+ file_count = len(files_with_matches)+ summary = f"Found {total_hits} Turbopuffer matches in {file_count} file{'s' if file_count != 1 else ''} for query: '{query}'"++ # combine all results+ formatted_results = [summary, "=" * len(summary)] + results++ self.logger.info(f"Turbopuffer search completed: {total_hits} matches across {file_count} files")+ return "\n".join(formatted_results)++ async def _search_files_pinecone(self, agent_state: AgentState, attached_sources: List[Source], query: str, limit: int) -> str:"""Search files using Pinecone vector database."""# Extract unique source_ids# TODO: Inefficient- attached_sources = await self.agent_manager.list_attached_sources_async(agent_id=agent_state.id, actor=self.actor)source_ids = [source.id for source in attached_sources]if not source_ids:return "No valid source IDs found for attached files"@@ -658,7 +783,7 @@ class LettaFileToolExecutor(ToolExecutor):self.logger.info(f"Pinecone search completed: {total_hits} matches across {file_count} files")return "\n".join(formatted_results)- async def _search_files_traditional(self, agent_state: AgentState, query: str, limit: int) -> str:+ async def _search_files_native(self, agent_state: AgentState, query: str, limit: int) -> str:"""Traditional search using existing passage manager."""# Get semantic search resultspassages = await self.agent_manager.query_source_passages_async(diff --git a/tests/test_file_processor.py b/tests/test_file_processor.pyindex 15ba50107..39dd790ac 100644--- a/tests/test_file_processor.py+++ b/tests/test_file_processor.py@@ -258,7 +258,7 @@ class TestFileProcessorWithPinecone:embedder = PineconeEmbedder()# Create file processor with Pinecone enabled- file_processor = FileProcessor(file_parser=file_parser, embedder=embedder, actor=mock_actor, using_pinecone=True)+ file_processor = FileProcessor(file_parser=file_parser, embedder=embedder, actor=mock_actor)# Track file manager update callsupdate_calls = []diff --git a/tests/test_sources.py b/tests/test_sources.pyindex 497405f6f..f71422d81 100644--- a/tests/test_sources.py+++ b/tests/test_sources.py@@ -13,6 +13,7 @@ from letta_client.types import AgentStatefrom letta.constants import DEFAULT_ORG_ID, FILES_TOOLSfrom letta.helpers.pinecone_utils import should_use_pinecone+from letta.helpers.tpuf_client import TurbopufferClientfrom letta.schemas.enums import FileProcessingStatus, ToolTypefrom letta.schemas.message import MessageCreatefrom letta.schemas.user import User@@ -95,7 +96,7 @@ def agent_state(disable_pinecone, client: LettaSDKClient):# Tests-def test_auto_attach_detach_files_tools(disable_pinecone, client: LettaSDKClient):+def test_auto_attach_detach_files_tools(disable_pinecone, disable_turbopuffer, client: LettaSDKClient):"""Test automatic attachment and detachment of file tools when managing agent sources."""# Create agent with basic configurationagent = client.agents.create(@@ -168,6 +169,7 @@ def test_auto_attach_detach_files_tools(disable_pinecone, client: LettaSDKClient)def test_file_upload_creates_source_blocks_correctly(disable_pinecone,+ disable_turbopuffer,client: LettaSDKClient,agent_state: AgentState,file_path: str,@@ -237,7 +239,9 @@ def test_file_upload_creates_source_blocks_correctly(settings.mistral_api_key = original_mistral_key-def test_attach_existing_files_creates_source_blocks_correctly(disable_pinecone, client: LettaSDKClient, agent_state: AgentState):+def test_attach_existing_files_creates_source_blocks_correctly(+ disable_pinecone, disable_turbopuffer, client: LettaSDKClient, agent_state: AgentState+):# Create a new sourcesource = client.sources.create(name="test_source", embedding="openai/text-embedding-3-small")assert len(client.sources.list()) == 1@@ -302,7 +306,9 @@ def test_attach_existing_files_creates_source_blocks_correctly(disable_pinecone,assert "<directories>" not in raw_system_message_after_detach-def test_delete_source_removes_source_blocks_correctly(disable_pinecone, client: LettaSDKClient, agent_state: AgentState):+def test_delete_source_removes_source_blocks_correctly(+ disable_pinecone, disable_turbopuffer, client: LettaSDKClient, agent_state: AgentState+):# Create a new sourcesource = client.sources.create(name="test_source", embedding="openai/text-embedding-3-small")assert len(client.sources.list()) == 1@@ -360,7 +366,7 @@ def test_delete_source_removes_source_blocks_correctly(disable_pinecone, client:assert not any("test" in b.value for b in blocks)-def test_agent_uses_open_close_file_correctly(disable_pinecone, client: LettaSDKClient, agent_state: AgentState):+def test_agent_uses_open_close_file_correctly(disable_pinecone, disable_turbopuffer, client: LettaSDKClient, agent_state: AgentState):# Create a new sourcesource = client.sources.create(name="test_source", embedding="openai/text-embedding-3-small")@@ -463,7 +469,7 @@ def test_agent_uses_open_close_file_correctly(disable_pinecone, client: LettaSDKprint("✓ File successfully opened with different range - content differs as expected")-def test_agent_uses_search_files_correctly(disable_pinecone, client: LettaSDKClient, agent_state: AgentState):+def test_agent_uses_search_files_correctly(disable_pinecone, disable_turbopuffer, client: LettaSDKClient, agent_state: AgentState):# Create a new sourcesource = client.sources.create(name="test_source", embedding="openai/text-embedding-3-small")@@ -509,7 +515,7 @@ def test_agent_uses_search_files_correctly(disable_pinecone, client: LettaSDKCliassert all(tr.status == "success" for tr in tool_returns), f"Tool call failed {tr}"-def test_agent_uses_grep_correctly_basic(disable_pinecone, client: LettaSDKClient, agent_state: AgentState):+def test_agent_uses_grep_correctly_basic(disable_pinecone, disable_turbopuffer, client: LettaSDKClient, agent_state: AgentState):# Create a new sourcesource = client.sources.create(name="test_source", embedding="openai/text-embedding-3-small")@@ -551,7 +557,7 @@ def test_agent_uses_grep_correctly_basic(disable_pinecone, client: LettaSDKClienassert all(tr.status == "success" for tr in tool_returns), "Tool call failed"-def test_agent_uses_grep_correctly_advanced(disable_pinecone, client: LettaSDKClient, agent_state: AgentState):+def test_agent_uses_grep_correctly_advanced(disable_pinecone, disable_turbopuffer, client: LettaSDKClient, agent_state: AgentState):# Create a new sourcesource = client.sources.create(name="test_source", embedding="openai/text-embedding-3-small")@@ -599,7 +605,7 @@ def test_agent_uses_grep_correctly_advanced(disable_pinecone, client: LettaSDKClassert "511:" in tool_return_message.tool_return-def test_create_agent_with_source_ids_creates_source_blocks_correctly(disable_pinecone, client: LettaSDKClient):+def test_create_agent_with_source_ids_creates_source_blocks_correctly(disable_pinecone, disable_turbopuffer, client: LettaSDKClient):"""Test that creating an agent with source_ids parameter correctly creates source blocks."""# Create a new sourcesource = client.sources.create(name="test_source", embedding="openai/text-embedding-3-small")@@ -642,7 +648,7 @@ def test_create_agent_with_source_ids_creates_source_blocks_correctly(disable_piassert file_tools == set(FILES_TOOLS)-def test_view_ranges_have_metadata(disable_pinecone, client: LettaSDKClient, agent_state: AgentState):+def test_view_ranges_have_metadata(disable_pinecone, disable_turbopuffer, client: LettaSDKClient, agent_state: AgentState):# Create a new sourcesource = client.sources.create(name="test_source", embedding="openai/text-embedding-3-small")@@ -705,7 +711,7 @@ def test_view_ranges_have_metadata(disable_pinecone, client: LettaSDKClient, age)-def test_duplicate_file_renaming(disable_pinecone, client: LettaSDKClient):+def test_duplicate_file_renaming(disable_pinecone, disable_turbopuffer, client: LettaSDKClient):"""Test that duplicate files are renamed with count-based suffixes (e.g., file.txt, file (1).txt, file (2).txt)"""# Create a new sourcesource = client.sources.create(name="test_duplicate_source", embedding="openai/text-embedding-3-small")@@ -744,7 +750,7 @@ def test_duplicate_file_renaming(disable_pinecone, client: LettaSDKClient):print(f" File {i + 1}: original='{file.original_file_name}' → renamed='{file.file_name}'")-def test_duplicate_file_handling_replace(disable_pinecone, client: LettaSDKClient):+def test_duplicate_file_handling_replace(disable_pinecone, disable_turbopuffer, client: LettaSDKClient):"""Test that DuplicateFileHandling.REPLACE replaces existing files with same name"""# Create a new sourcesource = client.sources.create(name="test_replace_source", embedding="openai/text-embedding-3-small")@@ -826,7 +832,7 @@ def test_duplicate_file_handling_replace(disable_pinecone, client: LettaSDKClienos.unlink(temp_file_path)-def test_upload_file_with_custom_name(disable_pinecone, client: LettaSDKClient):+def test_upload_file_with_custom_name(disable_pinecone, disable_turbopuffer, client: LettaSDKClient):"""Test that uploading a file with a custom name overrides the original filename"""# Create agentagent_state = client.agents.create(@@ -907,7 +913,7 @@ def test_upload_file_with_custom_name(disable_pinecone, client: LettaSDKClient):os.unlink(temp_file_path)-def test_open_files_schema_descriptions(disable_pinecone, client: LettaSDKClient):+def test_open_files_schema_descriptions(disable_pinecone, disable_turbopuffer, client: LettaSDKClient):"""Test that open_files tool schema contains correct descriptions from docstring"""# Get the open_files tool@@ -990,7 +996,7 @@ def test_open_files_schema_descriptions(disable_pinecone, client: LettaSDKClientassert length_prop["type"] == "integer"-def test_grep_files_schema_descriptions(disable_pinecone, client: LettaSDKClient):+def test_grep_files_schema_descriptions(disable_pinecone, disable_turbopuffer, client: LettaSDKClient):"""Test that grep_files tool schema contains correct descriptions from docstring"""# Get the grep_files tool@@ -1076,10 +1082,174 @@ def test_grep_files_schema_descriptions(disable_pinecone, client: LettaSDKClientassert "Navigation hint for next page if more matches exist" in description-# --- Pinecone Tests ---+def test_agent_open_file(disable_pinecone, disable_turbopuffer, client: LettaSDKClient, agent_state: AgentState):+ """Test client.agents.open_file() function"""+ # Create a new source+ source = client.sources.create(name="test_source", embedding="openai/text-embedding-3-small")++ # Attach source to agent+ client.agents.sources.attach(source_id=source.id, agent_id=agent_state.id)++ # Upload a file+ file_path = "tests/data/test.txt"+ file_metadata = upload_file_and_wait(client, source.id, file_path)++ # Basic test open_file function+ closed_files = client.agents.files.open(agent_id=agent_state.id, file_id=file_metadata.id)+ assert len(closed_files) == 0++ system = get_raw_system_message(client, agent_state.id)+ assert '<file status="open" name="test_source/test.txt">' in system+ assert "[Viewing file start (out of 1 lines)]" in system+++def test_agent_close_file(disable_pinecone, disable_turbopuffer, client: LettaSDKClient, agent_state: AgentState):+ """Test client.agents.close_file() function"""+ # Create a new source+ source = client.sources.create(name="test_source", embedding="openai/text-embedding-3-small")++ # Attach source to agent+ client.agents.sources.attach(source_id=source.id, agent_id=agent_state.id)++ # Upload a file+ file_path = "tests/data/test.txt"+ file_metadata = upload_file_and_wait(client, source.id, file_path)++ # First open the file+ client.agents.files.open(agent_id=agent_state.id, file_id=file_metadata.id)++ # Test close_file function+ client.agents.files.close(agent_id=agent_state.id, file_id=file_metadata.id)++ system = get_raw_system_message(client, agent_state.id)+ assert '<file status="closed" name="test_source/test.txt">' in system+++def test_agent_close_all_open_files(disable_pinecone, disable_turbopuffer, client: LettaSDKClient, agent_state: AgentState):+ """Test client.agents.close_all_open_files() function"""+ # Create a new source+ source = client.sources.create(name="test_source", embedding="openai/text-embedding-3-small")++ # Attach source to agent+ client.agents.sources.attach(source_id=source.id, agent_id=agent_state.id)++ # Upload multiple files+ file_paths = ["tests/data/test.txt", "tests/data/test.md"]+ file_metadatas = []+ for file_path in file_paths:+ file_metadata = upload_file_and_wait(client, source.id, file_path)+ file_metadatas.append(file_metadata)+ # Open each file+ client.agents.files.open(agent_id=agent_state.id, file_id=file_metadata.id)++ system = get_raw_system_message(client, agent_state.id)+ assert '<file status="open"' in system++ # Test close_all_open_files function+ result = client.agents.files.close_all(agent_id=agent_state.id)++ # Verify result is a list of strings+ assert isinstance(result, list), f"Expected list, got {type(result)}"+ assert all(isinstance(item, str) for item in result), "All items in result should be strings"++ system = get_raw_system_message(client, agent_state.id)+ assert '<file status="open"' not in system+++def test_file_processing_timeout(disable_pinecone, disable_turbopuffer, client: LettaSDKClient):+ """Test that files in non-terminal states are moved to error after timeout"""+ # Create a source+ source = client.sources.create(name="test_timeout_source", embedding="openai/text-embedding-3-small")+ # Upload a file+ file_path = "tests/data/test.txt"+ with open(file_path, "rb") as f:+ file_metadata = client.sources.files.upload(source_id=source.id, file=f)-def test_pinecone_search_files_tool(client: LettaSDKClient):+ # Get the file ID+ file_id = file_metadata.id++ # Test the is_terminal_state method directly (this doesn't require server mocking)+ assert FileProcessingStatus.COMPLETED.is_terminal_state() == True+ assert FileProcessingStatus.ERROR.is_terminal_state() == True+ assert FileProcessingStatus.PARSING.is_terminal_state() == False+ assert FileProcessingStatus.EMBEDDING.is_terminal_state() == False+ assert FileProcessingStatus.PENDING.is_terminal_state() == False++ # For testing the actual timeout logic, we can check the current file status+ current_file = client.sources.get_file_metadata(source_id=source.id, file_id=file_id)++ # Convert string status to enum for testing+ status_enum = FileProcessingStatus(current_file.processing_status)++ # Verify that files in terminal states are not affected by timeout checks+ if status_enum.is_terminal_state():+ # This is the expected behavior - files that completed processing shouldn't timeout+ print(f"File {file_id} is in terminal state: {current_file.processing_status}")+ assert status_enum in [FileProcessingStatus.COMPLETED, FileProcessingStatus.ERROR]+ else:+ # If file is still processing, it should eventually complete or timeout+ # In a real scenario, we'd wait and check, but for unit tests we just verify the logic exists+ print(f"File {file_id} is still processing: {current_file.processing_status}")+ assert status_enum in [FileProcessingStatus.PENDING, FileProcessingStatus.PARSING, FileProcessingStatus.EMBEDDING]+++def test_file_processing_timeout_logic():+ """Test the timeout logic directly without server dependencies"""+ from datetime import timezone++ # Test scenario: file created 35 minutes ago, timeout is 30 minutes+ old_time = datetime.now(timezone.utc) - timedelta(minutes=35)+ current_time = datetime.now(timezone.utc)+ timeout_minutes = 30++ # Calculate timeout threshold+ timeout_threshold = current_time - timedelta(minutes=timeout_minutes)++ # Verify timeout logic+ assert old_time < timeout_threshold, "File created 35 minutes ago should be past 30-minute timeout"++ # Test edge case: file created exactly at timeout+ edge_time = current_time - timedelta(minutes=timeout_minutes)+ assert not (edge_time < timeout_threshold), "File created exactly at timeout should not trigger timeout"++ # Test recent file+ recent_time = current_time - timedelta(minutes=10)+ assert not (recent_time < timeout_threshold), "Recent file should not trigger timeout"+++def test_letta_free_embedding(disable_pinecone, disable_turbopuffer, client: LettaSDKClient):+ """Test creating a source with letta/letta-free embedding and uploading a file"""+ # create a source with letta-free embedding+ source = client.sources.create(name="test_letta_free_source", embedding="letta/letta-free")++ # verify source was created with correct embedding+ assert source.name == "test_letta_free_source"+ print("\n\n\n\ntest")+ print(source.embedding_config)+ # assert source.embedding_config.embedding_model == "letta-free"++ # upload test.txt file+ file_path = "tests/data/test.txt"+ file_metadata = upload_file_and_wait(client, source.id, file_path)++ # verify file was uploaded successfully+ assert file_metadata.processing_status == "completed"+ assert file_metadata.source_id == source.id+ assert file_metadata.file_name == "test.txt"++ # verify file appears in source files list+ files = client.sources.files.list(source_id=source.id, limit=1)+ assert len(files) == 1+ assert files[0].id == file_metadata.id++ # cleanup+ client.sources.delete(source_id=source.id)+++# --- Pinecone Tests ---+def test_pinecone_search_files_tool(disable_turbopuffer, client: LettaSDKClient):"""Test that search_files tool uses Pinecone when enabled"""from letta.helpers.pinecone_utils import should_use_pinecone@@ -1130,7 +1300,7 @@ def test_pinecone_search_files_tool(client: LettaSDKClient):)-def test_pinecone_list_files_status(client: LettaSDKClient):+def test_pinecone_list_files_status(disable_turbopuffer, client: LettaSDKClient):"""Test that list_source_files properly syncs embedding status with Pinecone"""if not should_use_pinecone():pytest.skip("Pinecone not configured (missing API key or disabled), skipping Pinecone-specific tests")@@ -1165,7 +1335,7 @@ def test_pinecone_list_files_status(client: LettaSDKClient):client.sources.delete(source_id=source.id)-def test_pinecone_lifecycle_file_and_source_deletion(client: LettaSDKClient):+def test_pinecone_lifecycle_file_and_source_deletion(disable_turbopuffer, client: LettaSDKClient):"""Test that file and source deletion removes records from Pinecone"""from letta.helpers.pinecone_utils import list_pinecone_index_for_files, should_use_pinecone@@ -1236,167 +1406,196 @@ def test_pinecone_lifecycle_file_and_source_deletion(client: LettaSDKClient):)-def test_agent_open_file(disable_pinecone, client: LettaSDKClient, agent_state: AgentState):- """Test client.agents.open_file() function"""- # Create a new source- source = client.sources.create(name="test_source", embedding="openai/text-embedding-3-small")+# --- End Pinecone Tests ---- # Attach source to agent- client.agents.sources.attach(source_id=source.id, agent_id=agent_state.id)- # Upload a file- file_path = "tests/data/test.txt"- file_metadata = upload_file_and_wait(client, source.id, file_path)+# --- Turbopuffer Tests ---+def test_turbopuffer_search_files_tool(disable_pinecone, client: LettaSDKClient):+ """Test that search_files tool uses Turbopuffer when enabled"""+ agent = client.agents.create(+ name="test_turbopuffer_agent",+ memory_blocks=[+ CreateBlock(label="human", value="username: testuser"),+ ],+ model="openai/gpt-4o-mini",+ embedding="openai/text-embedding-3-small",+ )- # Basic test open_file function- closed_files = client.agents.files.open(agent_id=agent_state.id, file_id=file_metadata.id)- assert len(closed_files) == 0+ source = client.sources.create(name="test_turbopuffer_source", embedding="openai/text-embedding-3-small")+ client.agents.sources.attach(source_id=source.id, agent_id=agent.id)- system = get_raw_system_message(client, agent_state.id)- assert '<file status="open" name="test_source/test.txt">' in system- assert "[Viewing file start (out of 1 lines)]" in system+ file_path = "tests/data/long_test.txt"+ upload_file_and_wait(client, source.id, file_path)+ search_response = client.agents.messages.create(+ agent_id=agent.id,+ messages=[MessageCreate(role="user", content="Use the semantic_search_files tool to search for 'electoral history' in the files.")],+ )-def test_agent_close_file(disable_pinecone, client: LettaSDKClient, agent_state: AgentState):- """Test client.agents.close_file() function"""- # Create a new source- source = client.sources.create(name="test_source", embedding="openai/text-embedding-3-small")+ tool_calls = [msg for msg in search_response.messages if msg.message_type == "tool_call_message"]+ assert len(tool_calls) > 0, "No tool calls found"+ assert any(tc.tool_call.name == "semantic_search_files" for tc in tool_calls), "semantic_search_files not called"- # Attach source to agent- client.agents.sources.attach(source_id=source.id, agent_id=agent_state.id)+ tool_returns = [msg for msg in search_response.messages if msg.message_type == "tool_return_message"]+ assert len(tool_returns) > 0, "No tool returns found"+ assert all(tr.status == "success" for tr in tool_returns), "Tool call failed"- # Upload a file- file_path = "tests/data/test.txt"- file_metadata = upload_file_and_wait(client, source.id, file_path)+ search_results = tool_returns[0].tool_return+ print(f"Turbopuffer search results: {search_results}")+ assert "electoral" in search_results.lower() or "history" in search_results.lower(), (+ f"Search results should contain relevant content: {search_results}"+ )- # First open the file- client.agents.files.open(agent_id=agent_state.id, file_id=file_metadata.id)+ client.agents.delete(agent_id=agent.id)+ client.sources.delete(source_id=source.id)- # Test close_file function- client.agents.files.close(agent_id=agent_state.id, file_id=file_metadata.id)- system = get_raw_system_message(client, agent_state.id)- assert '<file status="closed" name="test_source/test.txt">' in system+def test_turbopuffer_file_processing_status(disable_pinecone, client: LettaSDKClient):+ """Test that file processing completes successfully with Turbopuffer"""+ print("Testing Turbopuffer file processing status")+ source = client.sources.create(name="test_tpuf_file_status", embedding="openai/text-embedding-3-small")-def test_agent_close_all_open_files(disable_pinecone, client: LettaSDKClient, agent_state: AgentState):- """Test client.agents.close_all_open_files() function"""- # Create a new source- source = client.sources.create(name="test_source", embedding="openai/text-embedding-3-small")+ file_paths = ["tests/data/long_test.txt", "tests/data/test.md"]+ uploaded_files = []+ for file_path in file_paths:+ file_metadata = upload_file_and_wait(client, source.id, file_path)+ uploaded_files.append(file_metadata)+ assert file_metadata.processing_status == "completed", f"File {file_path} should be completed"- # Attach source to agent- client.agents.sources.attach(source_id=source.id, agent_id=agent_state.id)+ files_list = client.sources.files.list(source_id=source.id, limit=100)++ assert len(files_list) == len(uploaded_files), f"Expected {len(uploaded_files)} files, got {len(files_list)}"++ for file_metadata in files_list:+ assert file_metadata.processing_status == "completed", f"File {file_metadata.file_name} should show completed status"++ if file_metadata.total_chunks and file_metadata.total_chunks > 0:+ assert file_metadata.chunks_embedded == file_metadata.total_chunks, (+ f"File {file_metadata.file_name} should have all chunks embedded: {file_metadata.chunks_embedded}/{file_metadata.total_chunks}"+ )++ client.sources.delete(source_id=source.id)+++def test_turbopuffer_lifecycle_file_and_source_deletion(disable_pinecone, client: LettaSDKClient):+ """Test that file and source deletion removes records from Turbopuffer"""+ source = client.sources.create(name="test_tpuf_lifecycle", embedding="openai/text-embedding-3-small")- # Upload multiple filesfile_paths = ["tests/data/test.txt", "tests/data/test.md"]- file_metadatas = []+ uploaded_files = []for file_path in file_paths:file_metadata = upload_file_and_wait(client, source.id, file_path)- file_metadatas.append(file_metadata)- # Open each file- client.agents.files.open(agent_id=agent_state.id, file_id=file_metadata.id)+ uploaded_files.append(file_metadata)- system = get_raw_system_message(client, agent_state.id)- assert '<file status="open"' in system+ user = User(name="temp", organization_id=DEFAULT_ORG_ID)+ tpuf_client = TurbopufferClient()- # Test close_all_open_files function- result = client.agents.files.close_all(agent_id=agent_state.id)+ # test file-level deletion+ if len(uploaded_files) > 1:+ file_to_delete = uploaded_files[0]- # Verify result is a list of strings- assert isinstance(result, list), f"Expected list, got {type(result)}"- assert all(isinstance(item, str) for item in result), "All items in result should be strings"+ passages_before = asyncio.run(+ tpuf_client.query_file_passages(+ source_ids=[source.id], organization_id=user.organization_id, actor=user, file_id=file_to_delete.id, top_k=100+ )+ )+ print(f"Found {len(passages_before)} passages for file before deletion")+ assert len(passages_before) > 0, "Should have passages before deletion"- system = get_raw_system_message(client, agent_state.id)- assert '<file status="open"' not in system+ client.sources.files.delete(source_id=source.id, file_id=file_to_delete.id)+ time.sleep(2)-def test_file_processing_timeout(disable_pinecone, client: LettaSDKClient):- """Test that files in non-terminal states are moved to error after timeout"""- # Create a source- source = client.sources.create(name="test_timeout_source", embedding="openai/text-embedding-3-small")+ passages_after = asyncio.run(+ tpuf_client.query_file_passages(+ source_ids=[source.id], organization_id=user.organization_id, actor=user, file_id=file_to_delete.id, top_k=100+ )+ )+ print(f"Found {len(passages_after)} passages for file after deletion")+ assert len(passages_after) == 0, f"File passages should be removed from Turbopuffer after deletion, but found {len(passages_after)}"- # Upload a file- file_path = "tests/data/test.txt"- with open(file_path, "rb") as f:- file_metadata = client.sources.files.upload(source_id=source.id, file=f)+ # test source-level deletion+ remaining_passages_before = []+ for file_metadata in uploaded_files[1:]:+ passages = asyncio.run(+ tpuf_client.query_file_passages(+ source_ids=[source.id], organization_id=user.organization_id, actor=user, file_id=file_metadata.id, top_k=100+ )+ )+ remaining_passages_before.extend(passages)- # Get the file ID- file_id = file_metadata.id+ print(f"Found {len(remaining_passages_before)} passages for remaining files before source deletion")+ assert len(remaining_passages_before) > 0, "Should have passages for remaining files"- # Test the is_terminal_state method directly (this doesn't require server mocking)- assert FileProcessingStatus.COMPLETED.is_terminal_state() == True- assert FileProcessingStatus.ERROR.is_terminal_state() == True- assert FileProcessingStatus.PARSING.is_terminal_state() == False- assert FileProcessingStatus.EMBEDDING.is_terminal_state() == False- assert FileProcessingStatus.PENDING.is_terminal_state() == False+ client.sources.delete(source_id=source.id)- # For testing the actual timeout logic, we can check the current file status- current_file = client.sources.get_file_metadata(source_id=source.id, file_id=file_id)+ time.sleep(3)- # Convert string status to enum for testing- status_enum = FileProcessingStatus(current_file.processing_status)+ remaining_passages_after = []+ for file_metadata in uploaded_files[1:]:+ try:+ passages = asyncio.run(+ tpuf_client.query_file_passages(+ source_ids=[source.id], organization_id=user.organization_id, actor=user, file_id=file_metadata.id, top_k=100+ )+ )+ remaining_passages_after.extend(passages)+ except Exception as e:+ print(f"Expected error querying deleted source: {e}")- # Verify that files in terminal states are not affected by timeout checks- if status_enum.is_terminal_state():- # This is the expected behavior - files that completed processing shouldn't timeout- print(f"File {file_id} is in terminal state: {current_file.processing_status}")- assert status_enum in [FileProcessingStatus.COMPLETED, FileProcessingStatus.ERROR]- else:- # If file is still processing, it should eventually complete or timeout- # In a real scenario, we'd wait and check, but for unit tests we just verify the logic exists- print(f"File {file_id} is still processing: {current_file.processing_status}")- assert status_enum in [FileProcessingStatus.PENDING, FileProcessingStatus.PARSING, FileProcessingStatus.EMBEDDING]+ print(f"Found {len(remaining_passages_after)} passages for files after source deletion")+ assert len(remaining_passages_after) == 0, (+ f"All source passages should be removed from Turbopuffer after source deletion, but found {len(remaining_passages_after)}"+ )-def test_file_processing_timeout_logic():- """Test the timeout logic directly without server dependencies"""- from datetime import timezone+def test_turbopuffer_multiple_sources(disable_pinecone, client: LettaSDKClient):+ """Test that Turbopuffer correctly isolates passages by source in org-scoped namespace"""+ source1 = client.sources.create(name="test_tpuf_source1", embedding="openai/text-embedding-3-small")+ source2 = client.sources.create(name="test_tpuf_source2", embedding="openai/text-embedding-3-small")- # Test scenario: file created 35 minutes ago, timeout is 30 minutes- old_time = datetime.now(timezone.utc) - timedelta(minutes=35)- current_time = datetime.now(timezone.utc)- timeout_minutes = 30+ file1_metadata = upload_file_and_wait(client, source1.id, "tests/data/test.txt")+ file2_metadata = upload_file_and_wait(client, source2.id, "tests/data/test.md")- # Calculate timeout threshold- timeout_threshold = current_time - timedelta(minutes=timeout_minutes)+ user = User(name="temp", organization_id=DEFAULT_ORG_ID)+ tpuf_client = TurbopufferClient()- # Verify timeout logic- assert old_time < timeout_threshold, "File created 35 minutes ago should be past 30-minute timeout"+ source1_passages = asyncio.run(+ tpuf_client.query_file_passages(source_ids=[source1.id], organization_id=user.organization_id, actor=user, top_k=100)+ )- # Test edge case: file created exactly at timeout- edge_time = current_time - timedelta(minutes=timeout_minutes)- assert not (edge_time < timeout_threshold), "File created exactly at timeout should not trigger timeout"+ source2_passages = asyncio.run(+ tpuf_client.query_file_passages(source_ids=[source2.id], organization_id=user.organization_id, actor=user, top_k=100)+ )- # Test recent file- recent_time = current_time - timedelta(minutes=10)- assert not (recent_time < timeout_threshold), "Recent file should not trigger timeout"+ print(f"Source1 has {len(source1_passages)} passages")+ print(f"Source2 has {len(source2_passages)} passages")+ assert len(source1_passages) > 0, "Source1 should have passages"+ assert len(source2_passages) > 0, "Source2 should have passages"-def test_letta_free_embedding(disable_pinecone, client: LettaSDKClient):- """Test creating a source with letta/letta-free embedding and uploading a file"""- # create a source with letta-free embedding- source = client.sources.create(name="test_letta_free_source", embedding="letta/letta-free")+ for passage, _, _ in source1_passages:+ assert passage.source_id == source1.id, f"Passage should belong to source1, but has source_id={passage.source_id}"+ assert passage.file_id == file1_metadata.id, f"Passage should belong to file1, but has file_id={passage.file_id}"- # verify source was created with correct embedding- assert source.name == "test_letta_free_source"- print("\n\n\n\ntest")- print(source.embedding_config)- # assert source.embedding_config.embedding_model == "letta-free"+ for passage, _, _ in source2_passages:+ assert passage.source_id == source2.id, f"Passage should belong to source2, but has source_id={passage.source_id}"+ assert passage.file_id == file2_metadata.id, f"Passage should belong to file2, but has file_id={passage.file_id}"- # upload test.txt file- file_path = "tests/data/test.txt"- file_metadata = upload_file_and_wait(client, source.id, file_path)+ # delete source1 and verify source2 is unaffected+ client.sources.delete(source_id=source1.id)+ time.sleep(2)- # verify file was uploaded successfully- assert file_metadata.processing_status == "completed"- assert file_metadata.source_id == source.id- assert file_metadata.file_name == "test.txt"+ source2_passages_after = asyncio.run(+ tpuf_client.query_file_passages(source_ids=[source2.id], organization_id=user.organization_id, actor=user, top_k=100)+ )- # verify file appears in source files list- files = client.sources.files.list(source_id=source.id, limit=1)- assert len(files) == 1- assert files[0].id == file_metadata.id+ assert len(source2_passages_after) == len(source2_passages), (+ f"Source2 should still have all passages after source1 deletion: {len(source2_passages_after)} vs {len(source2_passages)}"+ )- # cleanup- client.sources.delete(source_id=source.id)+ client.sources.delete(source_id=source2.id)+++# --- End Turbopuffer Tests ---