Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Parallel web search tool (#2890)
Matthew Zhoud5d71a776a44
diff --git a/letta/functions/function_sets/builtin.py b/letta/functions/function_sets/builtin.pyindex f06409a28..a7ff8fb3d 100644--- a/letta/functions/function_sets/builtin.py+++ b/letta/functions/function_sets/builtin.py@@ -1,4 +1,6 @@-from typing import Literal+from typing import List, Literal++from letta.functions.types import SearchTaskdef run_code(code: str, language: Literal["python", "js", "ts", "r", "java"]) -> str:@@ -16,29 +18,34 @@ def run_code(code: str, language: Literal["python", "js", "ts", "r", "java"]) ->async def web_search(- query: str,- question: str,- limit: int = 5,+ tasks: List[SearchTask],+ limit: int = 3,return_raw: bool = False,) -> str:"""- Search the web with the `query` and extract passages that answer the provided `question`.+ Search the web with a list of query/question pairs and extract passages that answer the corresponding questions.Examples:- query -> "Tesla Q1 2025 earnings report PDF"- question -> "What was Tesla's net profit in Q1 2025?"-- query -> "Letta API prebuilt tools core_memory_append"- question -> "What does the core_memory_append tool do in Letta?"+ tasks -> [+ SearchTask(+ query="Tesla Q1 2025 earnings report PDF",+ question="What was Tesla's net profit in Q1 2025?"+ ),+ SearchTask(+ query="Letta API prebuilt tools core_memory_append",+ question="What does the core_memory_append tool do in Letta?"+ )+ ]Args:- query (str): The raw web-search query.- question (str): The information goal to answer using the retrieved pages. Consider the context and intent of the conversation so far when forming the question.- limit (int, optional): Maximum number of URLs to fetch and analyse (must be > 0). Defaults to 5.- return_raw (bool, optional): If set to True, returns the raw content of the web page. This should be False unless otherwise specified by the user. Defaults to False.+ tasks (List[SearchTask]): A list of search tasks, each containing a `query` and a corresponding `question`.+ limit (int, optional): Maximum number of URLs to fetch and analyse per task (must be > 0). Defaults to 3.+ return_raw (bool, optional): If set to True, returns the raw content of the web pages.+ This should be False unless otherwise specified by the user. Defaults to False.Returns:- str: A JSON-encoded string containing ranked snippets with their source- URLs and relevance scores.+ str: A JSON-encoded string containing a list of search results.+ Each result includes ranked snippets with their source URLs and relevance scores,+ corresponding to each search task."""raise NotImplementedError("This is only available on the latest agent architecture. Please contact the Letta team.")diff --git a/letta/functions/prompts.py b/letta/functions/prompts.pyindex 85bd99a94..780280c35 100644--- a/letta/functions/prompts.py+++ b/letta/functions/prompts.py@@ -1,24 +1,26 @@-"""Prompts for Letta function tools."""+FIRECRAWL_SEARCH_SYSTEM_PROMPT = """You are an expert at extracting relevant information from web content.-FIRECRAWL_SEARCH_SYSTEM_PROMPT = """You are an expert information extraction assistant. Your task is to analyze a document and extract the most relevant passages that answer a specific question, based on a search query context.+Given a document with line numbers (format: "LINE_NUM: content"), identify passages that answer the provided question by returning line ranges:+- start_line: The starting line number (inclusive)+- end_line: The ending line number (inclusive)-Guidelines:-1. Extract substantial, lengthy text snippets that directly address the question-2. Preserve important context and details in each snippet - err on the side of including more rather than less-3. Keep thinking very brief (1 short sentence) - focus on WHY the snippet is relevant, not WHAT it says-4. Only extract snippets that actually answer or relate to the question - don't force relevance-5. Be comprehensive - include all relevant information, don't limit the number of snippets-6. Prioritize longer, information-rich passages over shorter ones"""+SELECTION PRINCIPLES:+1. Prefer comprehensive passages that include full context+2. Capture complete thoughts, examples, and explanations+3. When relevant content spans multiple paragraphs, include the entire section+4. Favor fewer, substantial passages over many fragments+Focus on passages that can stand alone as complete, meaningful responses."""-def get_firecrawl_search_user_prompt(query: str, question: str, markdown_content: str) -> str:- """Generate the user prompt for firecrawl search analysis."""++def get_firecrawl_search_user_prompt(query: str, question: str, numbered_content: str) -> str:+ """Generate the user prompt for line-number based search analysis."""return f"""Search Query: {query}Question to Answer: {question}-Document Content:-```markdown-{markdown_content}-```+Document Content (with line numbers):+{numbered_content}++Identify line ranges that best answer: "{question}"-Please analyze this document and extract all relevant passages that help answer the question."""+Select comprehensive passages with full context. Include entire sections when relevant."""diff --git a/letta/functions/types.py b/letta/functions/types.pynew file mode 100644index 000000000..420ef82ce--- /dev/null+++ b/letta/functions/types.py@@ -0,0 +1,6 @@+from pydantic import BaseModel, Field+++class SearchTask(BaseModel):+ query: str = Field(description="Search query for web search")+ question: str = Field(description="Question to answer from search results, considering full conversation context")diff --git a/letta/services/file_processor/chunker/llama_index_chunker.py b/letta/services/file_processor/chunker/llama_index_chunker.pyindex 94f45e0ae..dbb290e37 100644--- a/letta/services/file_processor/chunker/llama_index_chunker.py+++ b/letta/services/file_processor/chunker/llama_index_chunker.py@@ -1,4 +1,4 @@-from typing import List+from typing import List, Tuplefrom mistralai import OCRPageObject@@ -27,3 +27,91 @@ class LlamaIndexChunker:except Exception as e:logger.error(f"Chunking failed: {str(e)}")raise+++class MarkdownChunker:+ """Markdown-specific chunker that preserves line numbers for citation purposes"""++ def __init__(self, chunk_size: int = 2048):+ self.chunk_size = chunk_size+ # No overlap for line-based citations to avoid ambiguity++ from llama_index.core.node_parser import MarkdownNodeParser++ self.parser = MarkdownNodeParser()++ def chunk_markdown_with_line_numbers(self, markdown_content: str) -> List[Tuple[str, int, int]]:+ """+ Chunk markdown content while preserving line number mappings.++ Returns:+ List of tuples: (chunk_text, start_line, end_line)+ """+ try:+ # Split content into lines for line number tracking+ lines = markdown_content.split("\n")++ # Create nodes using MarkdownNodeParser+ from llama_index.core import Document++ document = Document(text=markdown_content)+ nodes = self.parser.get_nodes_from_documents([document])++ chunks_with_line_numbers = []++ for node in nodes:+ chunk_text = node.text++ # Find the line numbers for this chunk+ start_line, end_line = self._find_line_numbers(chunk_text, lines)++ chunks_with_line_numbers.append((chunk_text, start_line, end_line))++ return chunks_with_line_numbers++ except Exception as e:+ logger.error(f"Markdown chunking failed: {str(e)}")+ # Fallback to simple line-based chunking+ return self._fallback_line_chunking(markdown_content)++ def _find_line_numbers(self, chunk_text: str, lines: List[str]) -> Tuple[int, int]:+ """Find the start and end line numbers for a given chunk of text."""+ chunk_lines = chunk_text.split("\n")++ # Find the first line of the chunk in the original document+ start_line = 1+ for i, line in enumerate(lines):+ if chunk_lines[0].strip() in line.strip() and len(chunk_lines[0].strip()) > 10: # Avoid matching short lines+ start_line = i + 1+ break++ # Calculate end line+ end_line = start_line + len(chunk_lines) - 1++ return start_line, min(end_line, len(lines))++ def _fallback_line_chunking(self, markdown_content: str) -> List[Tuple[str, int, int]]:+ """Fallback chunking method that simply splits by lines with no overlap."""+ lines = markdown_content.split("\n")+ chunks = []++ i = 0+ while i < len(lines):+ chunk_lines = []+ start_line = i + 1+ char_count = 0++ # Build chunk until we hit size limit+ while i < len(lines) and char_count < self.chunk_size:+ line = lines[i]+ chunk_lines.append(line)+ char_count += len(line) + 1 # +1 for newline+ i += 1++ end_line = i+ chunk_text = "\n".join(chunk_lines)+ chunks.append((chunk_text, start_line, end_line))++ # No overlap - continue from where we left off++ return chunksdiff --git a/letta/services/tool_executor/builtin_tool_executor.py b/letta/services/tool_executor/builtin_tool_executor.pyindex dbeefdc8a..6753bba4e 100644--- a/letta/services/tool_executor/builtin_tool_executor.py+++ b/letta/services/tool_executor/builtin_tool_executor.py@@ -1,10 +1,12 @@import asyncioimport json+import timefrom typing import Any, Dict, List, Literal, Optionalfrom pydantic import BaseModelfrom letta.functions.prompts import FIRECRAWL_SEARCH_SYSTEM_PROMPT, get_firecrawl_search_user_prompt+from letta.functions.types import SearchTaskfrom letta.log import get_loggerfrom letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentState@@ -19,10 +21,16 @@ logger = get_logger(__name__)class Citation(BaseModel):- """A relevant text snippet extracted from a document."""+ """A relevant text snippet identified by line numbers in a document."""- text: str- thinking: str # Reasoning of why this snippet is relevant+ start_line: int # Starting line number (1-indexed)+ end_line: int # Ending line number (1-indexed, inclusive)+++class CitationWithText(BaseModel):+ """A citation with the actual extracted text."""++ text: str # The actual extracted text from the linesclass DocumentAnalysis(BaseModel):@@ -31,6 +39,12 @@ class DocumentAnalysis(BaseModel):citations: List[Citation]+class DocumentAnalysisWithText(BaseModel):+ """Analysis with extracted text from line citations."""++ citations: List[CitationWithText]++class LettaBuiltinToolExecutor(ToolExecutor):"""Executor for built in Letta tools."""@@ -88,39 +102,62 @@ class LettaBuiltinToolExecutor(ToolExecutor):out["error"] = errreturn out+ @trace_methodasync def web_search(self,agent_state: "AgentState",- query: str,- question: str,- limit: int = 5,+ tasks: List[SearchTask],+ limit: int = 3,return_raw: bool = False,) -> str:"""- Search the web with the `query` and extract passages that answer the provided `question`.+ Search the web with a list of query/question pairs and extract passages that answer the corresponding questions.Examples:- query -> "Tesla Q1 2025 earnings report PDF"- question -> "What was Tesla's net profit in Q1 2025?"-- query -> "Letta API prebuilt tools core_memory_append"- question -> "What does the core_memory_append tool do in Letta?"+ tasks -> [+ SearchTask(+ query="Tesla Q1 2025 earnings report PDF",+ question="What was Tesla's net profit in Q1 2025?"+ ),+ SearchTask(+ query="Letta API prebuilt tools core_memory_append",+ question="What does the core_memory_append tool do in Letta?"+ )+ ]Args:- query (str): The raw web-search query.- question (str): The information goal to answer using the retrieved pages.- limit (int, optional): Maximum number of URLs to fetch and analyse (must be > 0). Defaults to 5.- return_raw (bool, optional): If set to True, returns the raw content of the web page. This should be False unless otherwise specified by the user. Defaults to False.+ tasks (List[SearchTask]): A list of search tasks, each containing a `query` and a corresponding `question`.+ limit (int, optional): Maximum number of URLs to fetch and analyse per task (must be > 0). Defaults to 3.+ return_raw (bool, optional): If set to True, returns the raw content of the web pages.+ This should be False unless otherwise specified by the user. Defaults to False.Returns:- str: A JSON-encoded string containing ranked snippets with their source- URLs and relevance scores.+ str: A JSON-encoded string containing a list of search results.+ Each result includes ranked snippets with their source URLs and relevance scores,+ corresponding to each search task."""+ # TODO: Temporary, maybe deprecate this field?+ if return_raw:+ logger.warning("WARNING! return_raw was set to True, we default to False always. Deprecate this field.")+ return_raw = Falsetry:- from firecrawl import AsyncFirecrawlApp, ScrapeOptions+ from firecrawl import AsyncFirecrawlAppexcept ImportError:raise ImportError("firecrawl-py is not installed in the tool execution environment")+ if not tasks:+ return json.dumps({"error": "No search tasks provided."})++ # Convert dict objects to SearchTask objects+ search_tasks = []+ for task in tasks:+ if isinstance(task, dict):+ search_tasks.append(SearchTask(**task))+ else:+ search_tasks.append(task)++ logger.info(f"[DEBUG] Starting web search with {len(search_tasks)} tasks, limit={limit}, return_raw={return_raw}")+# Check if the API key exists on the agent stateagent_state_tool_env_vars = agent_state.get_agent_env_vars_as_dict()firecrawl_api_key = agent_state_tool_env_vars.get("FIRECRAWL_API_KEY") or tool_settings.firecrawl_api_key@@ -136,17 +173,64 @@ class LettaBuiltinToolExecutor(ToolExecutor):# Initialize Firecrawl clientapp = AsyncFirecrawlApp(api_key=firecrawl_api_key)- # Perform the search, just request markdown- search_result = await app.search(query, limit=limit, scrape_options=ScrapeOptions(formats=["markdown"]))+ # Process all search tasks in parallel+ search_task_coroutines = [self._process_single_search_task(app, task, limit, return_raw, api_key_source) for task in search_tasks]++ # Execute all searches concurrently+ search_results = await asyncio.gather(*search_task_coroutines, return_exceptions=True)++ # Build final response as a mapping of query -> result+ final_results = {}+ successful_tasks = 0+ failed_tasks = 0++ for i, result in enumerate(search_results):+ query = search_tasks[i].query+ if isinstance(result, Exception):+ logger.error(f"Search task {i} failed: {result}")+ failed_tasks += 1+ final_results[query] = {"query": query, "question": search_tasks[i].question, "error": str(result)}+ else:+ successful_tasks += 1+ final_results[query] = result++ logger.info(f"[DEBUG] Web search completed: {successful_tasks} successful, {failed_tasks} failed")++ # Build final response with api_key_source at top level+ response = {"api_key_source": api_key_source, "results": final_results}++ return json.dumps(response, indent=2, ensure_ascii=False)++ @trace_method+ async def _process_single_search_task(+ self, app: "AsyncFirecrawlApp", task: SearchTask, limit: int, return_raw: bool, api_key_source: str+ ) -> Dict[str, Any]:+ """Process a single search task."""+ from firecrawl import ScrapeOptions++ logger.info(f"[DEBUG] Starting Firecrawl search for query: '{task.query}' with limit={limit}")++ # Perform the search for this task+ search_result = await app.search(task.query, limit=limit, scrape_options=ScrapeOptions(formats=["markdown"]))++ logger.info(+ f"[DEBUG] Firecrawl search completed for '{task.query}': {len(search_result.get('data', [])) if search_result else 0} results"+ )if not search_result or not search_result.get("data"):- return json.dumps({"error": "No search results found."})+ return {"query": task.query, "question": task.question, "error": "No search results found."}++ # If raw results requested, return them directly+ if return_raw:+ return {"query": task.query, "question": task.question, "raw_results": search_result}# Check if OpenAI API key is available for semantic parsing- if not return_raw and model_settings.openai_api_key:+ if model_settings.openai_api_key:try:from openai import AsyncOpenAI+ logger.info(f"[DEBUG] Starting OpenAI analysis for '{task.query}'")+# Initialize OpenAI clientclient = AsyncOpenAI(api_key=model_settings.openai_api_key,@@ -160,15 +244,19 @@ class LettaBuiltinToolExecutor(ToolExecutor):for result in search_result.get("data"):if result.get("markdown"):# Create async task for OpenAI analysis- task = self._analyze_document_with_openai(client, result["markdown"], query, question)- analysis_tasks.append(task)+ analysis_task = self._analyze_document_with_openai(client, result["markdown"], task.query, task.question)+ analysis_tasks.append(analysis_task)results_with_markdown.append(result)else:results_without_markdown.append(result)+ logger.info(f"[DEBUG] Starting parallel OpenAI analysis of {len(analysis_tasks)} documents for '{task.query}'")+# Fire off all OpenAI requests concurrentlyanalyses = await asyncio.gather(*analysis_tasks, return_exceptions=True)+ logger.info(f"[DEBUG] Completed parallel OpenAI analysis of {len(analyses)} documents for '{task.query}'")+# Build processed resultsprocessed_results = []@@ -176,16 +264,21 @@ class LettaBuiltinToolExecutor(ToolExecutor):for result, analysis in zip(results_with_markdown, analyses):if isinstance(analysis, Exception) or analysis is None:logger.error(f"Analysis failed for {result.get('url')}, falling back to raw results")- return str(search_result)+ return {"query": task.query, "question": task.question, "raw_results": search_result}# All analyses succeeded, build processed resultsfor result, analysis in zip(results_with_markdown, analyses):+ # Extract actual text from line number citations+ analysis_with_text = None+ if analysis and analysis.citations:+ analysis_with_text = self._extract_text_from_line_citations(analysis, result["markdown"])+processed_results.append({"url": result.get("url"),"title": result.get("title"),"description": result.get("description"),- "analysis": analysis.model_dump() if analysis else None,+ "analysis": analysis_with_text.model_dump() if analysis_with_text else None,})@@ -195,35 +288,95 @@ class LettaBuiltinToolExecutor(ToolExecutor):{"url": result.get("url"), "title": result.get("title"), "description": result.get("description"), "analysis": None})- # Concatenate all relevant snippets into a final response- final_response = self._build_final_response(processed_results, query, question, api_key_source)- return final_response+ # Build final response for this task+ return self._build_final_response_dict(processed_results, task.query, task.question)except Exception as e:# Log error but continue with raw results- logger.error(f"Error with OpenAI processing: {e}")+ logger.error(f"Error with OpenAI processing for task '{task.query}': {e}")# Return raw search results if OpenAI processing isn't available or fails- return str(search_result)+ return {"query": task.query, "question": task.question, "raw_results": search_result}+ @trace_methodasync def _analyze_document_with_openai(self, client, markdown_content: str, query: str, question: str) -> Optional[DocumentAnalysis]:- """Use OpenAI to analyze a document and extract relevant passages."""- max_content_length = 200000 # GPT-4.1 has ~1M token context window, so we can be more generous with content length- if len(markdown_content) > max_content_length:- markdown_content = markdown_content[:max_content_length] + "..."+ """Use OpenAI to analyze a document and extract relevant passages using line numbers."""+ original_length = len(markdown_content)++ # Create numbered markdown for the LLM to reference+ numbered_lines = markdown_content.split("\n")+ numbered_markdown = "\n".join([f"{i+1:4d}: {line}" for i, line in enumerate(numbered_lines)])- user_prompt = get_firecrawl_search_user_prompt(query, question, markdown_content)+ # Truncate if too long+ max_content_length = 200000+ truncated = False+ if len(numbered_markdown) > max_content_length:+ numbered_markdown = numbered_markdown[:max_content_length] + "..."+ truncated = True++ user_prompt = get_firecrawl_search_user_prompt(query, question, numbered_markdown)++ logger.info(+ f"[DEBUG] Starting OpenAI request with line numbers - Query: '{query}', Content: {original_length} chars (truncated: {truncated})"+ )++ # Time the OpenAI request+ start_time = time.time()response = await client.beta.chat.completions.parse(model="gpt-4.1-mini-2025-04-14",messages=[{"role": "system", "content": FIRECRAWL_SEARCH_SYSTEM_PROMPT}, {"role": "user", "content": user_prompt}],response_format=DocumentAnalysis,temperature=0.1,+ max_tokens=300, # Limit output tokens - only need line numbers)- return response.choices[0].message.parsed+ end_time = time.time()+ request_duration = end_time - start_time++ # Get usage statistics and output length+ usage = response.usage+ parsed_result = response.choices[0].message.parsed+ num_citations = len(parsed_result.citations) if parsed_result else 0++ # Calculate output length (minimal now - just line numbers)+ output_length = 0+ if parsed_result and parsed_result.citations:+ for citation in parsed_result.citations:+ output_length += 20 # ~20 chars for line numbers only- def _build_final_response(self, processed_results: List[Dict], query: str, question: str, api_key_source: str = None) -> str:- """Build the final JSON response from all processed results."""+ logger.info(f"[TIMING] OpenAI request completed in {request_duration:.2f}s - Query: '{query}'")+ logger.info(f"[TOKENS] Total: {usage.total_tokens} (prompt: {usage.prompt_tokens}, completion: {usage.completion_tokens})")+ logger.info(f"[OUTPUT] Citations: {num_citations}, Output chars: {output_length} (line-number based)")++ return parsed_result++ def _extract_text_from_line_citations(self, analysis: DocumentAnalysis, original_markdown: str) -> DocumentAnalysisWithText:+ """Extract actual text from line number citations."""+ lines = original_markdown.split("\n")+ citations_with_text = []++ for citation in analysis.citations:+ try:+ # Convert to 0-indexed and ensure bounds+ start_idx = max(0, citation.start_line - 1)+ end_idx = min(len(lines), citation.end_line)++ # Extract the lines+ extracted_lines = lines[start_idx:end_idx]+ extracted_text = "\n".join(extracted_lines)++ citations_with_text.append(CitationWithText(text=extracted_text))++ except Exception as e:+ logger.info(f"[DEBUG] Failed to extract text for citation lines {citation.start_line}-{citation.end_line}: {e}")+ # Fall back to including the citation with empty text+ citations_with_text.append(CitationWithText(text=""))++ return DocumentAnalysisWithText(citations=citations_with_text)++ @trace_method+ def _build_final_response_dict(self, processed_results: List[Dict], query: str, question: str) -> Dict[str, Any]:+ """Build the final response dictionary from all processed results."""# Build sources arraysources = []@@ -250,11 +403,7 @@ class LettaBuiltinToolExecutor(ToolExecutor):"sources": sources,}- # Add API key source if provided- if api_key_source:- response["api_key_source"] = api_key_source-if total_snippets == 0:response["message"] = "No relevant passages found that directly answer the question."- return json.dumps(response, indent=2, ensure_ascii=False)+ return response