Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Add bulk source upsert mechanism (#3456)
Matthew Zhoue68f563b03bf
diff --git a/letta/services/agent_file_manager.py b/letta/services/agent_file_manager.pyindex 37a6affd4..78f34b896 100644--- a/letta/services/agent_file_manager.py+++ b/letta/services/agent_file_manager.py@@ -386,14 +386,27 @@ class AgentFileManager:file_to_db_ids[block_schema.id] = created_block.idimported_count += 1- # 3. Create sources (no dependencies)- for source_schema in schema.sources:- # Convert SourceSchema back to Source- source_data = source_schema.model_dump(exclude={"id", "embedding", "embedding_chunk_size"})- source = Source(**source_data)- created_source = await self.source_manager.create_source(source, actor)- file_to_db_ids[source_schema.id] = created_source.id- imported_count += 1+ # 3. Create sources (no dependencies) - using bulk upsert for efficiency+ if schema.sources:+ # convert source schemas to pydantic sources+ pydantic_sources = []+ for source_schema in schema.sources:+ source_data = source_schema.model_dump(exclude={"id", "embedding", "embedding_chunk_size"})+ pydantic_sources.append(Source(**source_data))++ # bulk upsert all sources at once+ created_sources = await self.source_manager.bulk_upsert_sources_async(pydantic_sources, actor)++ # map file ids to database ids+ # note: sources are matched by name during upsert, so we need to match by name here too+ created_sources_by_name = {source.name: source for source in created_sources}+ for source_schema in schema.sources:+ created_source = created_sources_by_name.get(source_schema.name)+ if created_source:+ file_to_db_ids[source_schema.id] = created_source.id+ imported_count += 1+ else:+ logger.warning(f"Source {source_schema.name} was not created during bulk upsert")# 4. Create files (depends on sources)for file_schema in schema.files:diff --git a/letta/services/source_manager.py b/letta/services/source_manager.pyindex e3287130c..967869199 100644--- a/letta/services/source_manager.py+++ b/letta/services/source_manager.py@@ -58,6 +58,123 @@ class SourceManager:await source.create_async(session, actor=actor)return source.to_pydantic()+ @enforce_types+ @trace_method+ async def bulk_upsert_sources_async(self, pydantic_sources: List[PydanticSource], actor: PydanticUser) -> List[PydanticSource]:+ """+ Bulk create or update multiple sources in a single database transaction.++ Uses optimized PostgreSQL bulk upsert when available, falls back to individual+ upserts for SQLite. This is much more efficient than calling create_source+ in a loop.++ IMPORTANT BEHAVIOR NOTES:+ - Sources are matched by (name, organization_id) unique constraint, NOT by ID+ - If a source with the same name already exists for the organization, it will be updated+ regardless of any ID provided in the input source+ - The existing source's ID is preserved during updates+ - If you provide a source with an explicit ID but a name that matches an existing source,+ the existing source will be updated and the provided ID will be ignored+ - This matches the behavior of create_source which also checks by ID first++ PostgreSQL optimization:+ - Uses native ON CONFLICT (name, organization_id) DO UPDATE for atomic upserts+ - All sources are processed in a single SQL statement for maximum efficiency++ SQLite fallback:+ - Falls back to individual create_source calls+ - Still benefits from batched transaction handling++ Args:+ pydantic_sources: List of sources to create or update+ actor: User performing the action++ Returns:+ List of created/updated sources+ """+ if not pydantic_sources:+ return []++ from letta.settings import settings++ if settings.letta_pg_uri_no_default:+ # use optimized postgresql bulk upsert+ async with db_registry.async_session() as session:+ return await self._bulk_upsert_postgresql(session, pydantic_sources, actor)+ else:+ # fallback to individual upserts for sqlite+ return await self._upsert_sources_individually(pydantic_sources, actor)++ @trace_method+ async def _bulk_upsert_postgresql(self, session, source_data_list: List[PydanticSource], actor: PydanticUser) -> List[PydanticSource]:+ """Hyper-optimized PostgreSQL bulk upsert using ON CONFLICT DO UPDATE."""+ from sqlalchemy import func, select+ from sqlalchemy.dialects.postgresql import insert++ # prepare data for bulk insert+ table = SourceModel.__table__+ valid_columns = {col.name for col in table.columns}++ insert_data = []+ for source in source_data_list:+ source_dict = source.model_dump(to_orm=True)+ # set created/updated by fields++ if actor:+ source_dict["_created_by_id"] = actor.id+ source_dict["_last_updated_by_id"] = actor.id+ source_dict["organization_id"] = actor.organization_id++ # filter to only include columns that exist in the table+ filtered_dict = {k: v for k, v in source_dict.items() if k in valid_columns}+ insert_data.append(filtered_dict)++ # use postgresql's native bulk upsert+ stmt = insert(table).values(insert_data)++ # on conflict, update all columns except id, created_at, and _created_by_id+ excluded = stmt.excluded+ update_dict = {}+ for col in table.columns:+ if col.name not in ("id", "created_at", "_created_by_id"):+ if col.name == "updated_at":+ update_dict[col.name] = func.now()+ else:+ update_dict[col.name] = excluded[col.name]++ upsert_stmt = stmt.on_conflict_do_update(index_elements=["name", "organization_id"], set_=update_dict)++ await session.execute(upsert_stmt)+ await session.commit()++ # fetch results+ source_names = [source.name for source in source_data_list]+ result_query = select(SourceModel).where(+ SourceModel.name.in_(source_names), SourceModel.organization_id == actor.organization_id, SourceModel.is_deleted == False+ )+ result = await session.execute(result_query)+ return [source.to_pydantic() for source in result.scalars()]++ @trace_method+ async def _upsert_sources_individually(self, source_data_list: List[PydanticSource], actor: PydanticUser) -> List[PydanticSource]:+ """Fallback to individual upserts for SQLite."""+ sources = []+ for source in source_data_list:+ # try to get existing source by name+ existing_source = await self.get_source_by_name(source.name, actor)+ if existing_source:+ # update existing source+ from letta.schemas.source import SourceUpdate++ update_data = source.model_dump(exclude={"id"}, exclude_none=True)+ updated_source = await self.update_source(existing_source.id, SourceUpdate(**update_data), actor)+ sources.append(updated_source)+ else:+ # create new source+ created_source = await self.create_source(source, actor)+ sources.append(created_source)+ return sources+@enforce_types@trace_methodasync def update_source(self, source_id: str, source_update: SourceUpdate, actor: PydanticUser) -> PydanticSource:diff --git a/letta/services/tool_executor/builtin_tool_executor.py b/letta/services/tool_executor/builtin_tool_executor.pyindex d8cd24a6b..e7415cac9 100644--- a/letta/services/tool_executor/builtin_tool_executor.py+++ b/letta/services/tool_executor/builtin_tool_executor.py@@ -1,6 +1,5 @@import asyncioimport json-import osimport timefrom typing import Any, Dict, List, Literal, Optional@@ -176,7 +175,9 @@ class LettaBuiltinToolExecutor(ToolExecutor):app = AsyncFirecrawlApp(api_key=firecrawl_api_key)# Process all search tasks in parallel- search_task_coroutines = [self._process_single_search_task(app, task, limit, return_raw, api_key_source) for task in search_tasks]+ search_task_coroutines = [+ self._process_single_search_task(app, task, limit, return_raw, api_key_source, agent_state) for task in search_tasks+ ]# Execute all searches concurrentlysearch_results = await asyncio.gather(*search_task_coroutines, return_exceptions=True)@@ -205,7 +206,7 @@ class LettaBuiltinToolExecutor(ToolExecutor):@trace_methodasync def _process_single_search_task(- self, app: "AsyncFirecrawlApp", task: SearchTask, limit: int, return_raw: bool, api_key_source: str+ self, app: "AsyncFirecrawlApp", task: SearchTask, limit: int, return_raw: bool, api_key_source: str, agent_state: "AgentState") -> Dict[str, Any]:"""Process a single search task."""from firecrawl import ScrapeOptions@@ -246,7 +247,9 @@ class LettaBuiltinToolExecutor(ToolExecutor):for result in search_result.get("data"):if result.get("markdown"):# Create async task for OpenAI analysis- analysis_task = self._analyze_document_with_openai(client, result["markdown"], task.query, task.question)+ analysis_task = self._analyze_document_with_openai(+ client, result["markdown"], task.query, task.question, agent_state+ )analysis_tasks.append(analysis_task)results_with_markdown.append(result)else:@@ -300,7 +303,9 @@ class LettaBuiltinToolExecutor(ToolExecutor):return {"query": task.query, "question": task.question, "raw_results": search_result}@trace_method- async def _analyze_document_with_openai(self, client, markdown_content: str, query: str, question: str) -> Optional[DocumentAnalysis]:+ async def _analyze_document_with_openai(+ self, client, markdown_content: str, query: str, question: str, agent_state: "AgentState"+ ) -> Optional[DocumentAnalysis]:"""Use OpenAI to analyze a document and extract relevant passages using line numbers."""original_length = len(markdown_content)@@ -324,7 +329,9 @@ class LettaBuiltinToolExecutor(ToolExecutor):# Time the OpenAI requeststart_time = time.time()- model = os.getenv(WEB_SEARCH_MODEL_ENV_VAR_NAME, WEB_SEARCH_MODEL_ENV_VAR_DEFAULT_VALUE)+ # Check agent state env vars first, then fall back to os.getenv+ agent_state_tool_env_vars = agent_state.get_agent_env_vars_as_dict()+ model = agent_state_tool_env_vars.get(WEB_SEARCH_MODEL_ENV_VAR_NAME) or WEB_SEARCH_MODEL_ENV_VAR_DEFAULT_VALUElogger.info(f"Using model {model} for web search result parsing")response = await client.beta.chat.completions.parse(model=model,diff --git a/tests/integration_test_builtin_tools.py b/tests/integration_test_builtin_tools.pyindex 4e8538257..83af3608e 100644--- a/tests/integration_test_builtin_tools.py+++ b/tests/integration_test_builtin_tools.py@@ -321,6 +321,7 @@ async def test_web_search_uses_agent_env_var_model():patch("openai.AsyncOpenAI") as mock_openai_class,patch("letta.services.tool_executor.builtin_tool_executor.model_settings") as mock_model_settings,patch.dict(os.environ, {WEB_SEARCH_MODEL_ENV_VAR_NAME: "gpt-4o"}),+ patch("firecrawl.AsyncFirecrawlApp") as mock_firecrawl_class,):# setup mocks@@ -330,6 +331,23 @@ async def test_web_search_uses_agent_env_var_model():mock_openai_class.return_value = mock_openai_clientmock_openai_client.beta.chat.completions.parse.return_value = mock_openai_response+ # Mock Firecrawl+ mock_firecrawl_app = AsyncMock()+ mock_firecrawl_class.return_value = mock_firecrawl_app++ # Mock search results with markdown content+ mock_search_result = {+ "data": [+ {+ "url": "https://example.com/test",+ "title": "Test Result",+ "description": "Test description",+ "markdown": "This is test markdown content for the search result.",+ }+ ]+ }+ mock_firecrawl_app.search.return_value = mock_search_result+# create executor with mock dependenciesexecutor = LettaBuiltinToolExecutor(message_manager=MagicMock(),diff --git a/tests/test_managers.py b/tests/test_managers.pyindex 5caf25e4b..ffc583c05 100644--- a/tests/test_managers.py+++ b/tests/test_managers.py@@ -5262,6 +5262,164 @@ async def test_update_source_no_changes(server: SyncServer, default_user):assert updated_source.description == source.description+async def test_bulk_upsert_sources_async(server: SyncServer, default_user):+ """Test bulk upserting sources."""+ sources_data = [+ PydanticSource(+ name="Bulk Source 1",+ description="First bulk source",+ embedding_config=DEFAULT_EMBEDDING_CONFIG,+ ),+ PydanticSource(+ name="Bulk Source 2",+ description="Second bulk source",+ embedding_config=DEFAULT_EMBEDDING_CONFIG,+ ),+ PydanticSource(+ name="Bulk Source 3",+ description="Third bulk source",+ embedding_config=DEFAULT_EMBEDDING_CONFIG,+ ),+ ]++ # Bulk upsert sources+ created_sources = await server.source_manager.bulk_upsert_sources_async(sources_data, default_user)++ # Verify all sources were created+ assert len(created_sources) == 3++ # Verify source details+ created_names = {source.name for source in created_sources}+ expected_names = {"Bulk Source 1", "Bulk Source 2", "Bulk Source 3"}+ assert created_names == expected_names++ # Verify organization assignment+ for source in created_sources:+ assert source.organization_id == default_user.organization_id+++async def test_bulk_upsert_sources_name_conflict(server: SyncServer, default_user):+ """Test bulk upserting sources with name conflicts."""+ # Create an existing source+ existing_source = await server.source_manager.create_source(+ PydanticSource(+ name="Existing Source",+ description="Already exists",+ embedding_config=DEFAULT_EMBEDDING_CONFIG,+ ),+ default_user,+ )++ # Try to bulk upsert with the same name+ sources_data = [+ PydanticSource(+ name="Existing Source", # Same name as existing+ description="Updated description",+ metadata={"updated": True},+ embedding_config=DEFAULT_EMBEDDING_CONFIG,+ ),+ PydanticSource(+ name="New Bulk Source",+ description="Completely new",+ embedding_config=DEFAULT_EMBEDDING_CONFIG,+ ),+ ]++ # Bulk upsert should update existing and create new+ result_sources = await server.source_manager.bulk_upsert_sources_async(sources_data, default_user)++ # Should return 2 sources+ assert len(result_sources) == 2++ # Find the updated source+ updated_source = next(s for s in result_sources if s.name == "Existing Source")++ # Verify the existing source was updated, not replaced+ assert updated_source.id == existing_source.id # ID should be preserved+ assert updated_source.description == "Updated description"+ assert updated_source.metadata == {"updated": True}++ # Verify new source was created+ new_source = next(s for s in result_sources if s.name == "New Bulk Source")+ assert new_source.description == "Completely new"+++async def test_bulk_upsert_sources_mixed_create_update(server: SyncServer, default_user):+ """Test bulk upserting with a mix of creates and updates."""+ # Create some existing sources+ existing1 = await server.source_manager.create_source(+ PydanticSource(+ name="Mixed Source 1",+ description="Original 1",+ embedding_config=DEFAULT_EMBEDDING_CONFIG,+ ),+ default_user,+ )+ existing2 = await server.source_manager.create_source(+ PydanticSource(+ name="Mixed Source 2",+ description="Original 2",+ embedding_config=DEFAULT_EMBEDDING_CONFIG,+ ),+ default_user,+ )++ # Bulk upsert with updates and new sources+ sources_data = [+ PydanticSource(+ name="Mixed Source 1", # Update existing+ description="Updated 1",+ instructions="New instructions 1",+ embedding_config=DEFAULT_EMBEDDING_CONFIG,+ ),+ PydanticSource(+ name="Mixed Source 3", # Create new+ description="New 3",+ embedding_config=DEFAULT_EMBEDDING_CONFIG,+ ),+ PydanticSource(+ name="Mixed Source 2", # Update existing+ description="Updated 2",+ metadata={"version": 2},+ embedding_config=DEFAULT_EMBEDDING_CONFIG,+ ),+ PydanticSource(+ name="Mixed Source 4", # Create new+ description="New 4",+ embedding_config=DEFAULT_EMBEDDING_CONFIG,+ ),+ ]++ # Perform bulk upsert+ result_sources = await server.source_manager.bulk_upsert_sources_async(sources_data, default_user)++ # Should return 4 sources+ assert len(result_sources) == 4++ # Verify updates preserved IDs+ source1 = next(s for s in result_sources if s.name == "Mixed Source 1")+ assert source1.id == existing1.id+ assert source1.description == "Updated 1"+ assert source1.instructions == "New instructions 1"++ source2 = next(s for s in result_sources if s.name == "Mixed Source 2")+ assert source2.id == existing2.id+ assert source2.description == "Updated 2"+ assert source2.metadata == {"version": 2}++ # Verify new sources were created+ source3 = next(s for s in result_sources if s.name == "Mixed Source 3")+ assert source3.description == "New 3"+ assert source3.id != existing1.id and source3.id != existing2.id++ source4 = next(s for s in result_sources if s.name == "Mixed Source 4")+ assert source4.description == "New 4"+ assert source4.id != existing1.id and source4.id != existing2.id++# ======================================================================================================================# Source Manager Tests - Files# ======================================================================================================================