Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
chore: merge oss (#3712)
Sarah Woodersbde471429414
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.mdindex ff63f2ac0..4c6269403 100644--- a/.github/ISSUE_TEMPLATE/bug_report.md+++ b/.github/ISSUE_TEMPLATE/bug_report.md@@ -11,20 +11,25 @@ assignees: ''A clear and concise description of what the bug is.**Please describe your setup**-- [ ] How did you install letta?- - `pip install letta`? `pip install letta-nightly`? `git clone`?+- [ ] How are you running Letta?+ - Docker+ - pip (legacy)+ - From source+ - Desktop- [ ] Describe your setup- What's your OS (Windows/MacOS/Linux)?- - How are you running `letta`? (`cmd.exe`/Powershell/Anaconda Shell/Terminal)+ - What is your `docker run ...` command (if applicable)**Screenshots**If applicable, add screenshots to help explain your problem.**Additional context**Add any other context about the problem here.+- What model you are using++**Agent File (optional)**+Please attach your `.af` file, as this helps with reproducing issues.-**Letta Config**-Please attach your `~/.letta/config` file or copy paste it below.---diff --git a/.github/scripts/model-sweep/conftest.py b/.github/scripts/model-sweep/conftest.pynew file mode 100644index 000000000..db66a444c--- /dev/null+++ b/.github/scripts/model-sweep/conftest.py@@ -0,0 +1,286 @@+import logging+import os+import socket+import threading+import time+from datetime import datetime, timezone+from typing import Generator++import pytest+import requests+from anthropic.types.beta.messages import BetaMessageBatch, BetaMessageBatchRequestCounts+from dotenv import load_dotenv+from letta_client import AsyncLetta, Letta++from letta.schemas.agent import AgentState+from letta.schemas.llm_config import LLMConfig+from letta.services.organization_manager import OrganizationManager+from letta.services.user_manager import UserManager+from letta.settings import tool_settings+++def pytest_configure(config):+ logging.basicConfig(level=logging.DEBUG)+++def disable_e2b_api_key() -> Generator[None, None, None]:+ """+ Temporarily disables the E2B API key by setting `tool_settings.e2b_api_key` to None+ for the duration of the test. Restores the original value afterward.+ """+ from letta.settings import tool_settings++ original_api_key = tool_settings.e2b_api_key+ tool_settings.e2b_api_key = None+ yield+ tool_settings.e2b_api_key = original_api_key+++def check_e2b_key_is_set():+ from letta.settings import tool_settings++ original_api_key = tool_settings.e2b_api_key+ assert original_api_key is not None, "Missing e2b key! Cannot execute these tests."+ yield+++def default_organization():+ """Fixture to create and return the default organization."""+ manager = OrganizationManager()+ org = manager.create_default_organization()+ yield org+++def default_user(default_organization):+ """Fixture to create and return the default user within the default organization."""+ manager = UserManager()+ user = manager.create_default_user(org_id=default_organization.id)+ yield user+++def check_composio_key_set():+ original_api_key = tool_settings.composio_api_key+ assert original_api_key is not None, "Missing composio key! Cannot execute this test."+ yield+++# --- Tool Fixtures ---+def weather_tool_func():+ def get_weather(location: str) -> str:+ """+ Fetches the current weather for a given location.++ Parameters:+ location (str): The location to get the weather for.++ Returns:+ str: A formatted string describing the weather in the given location.++ Raises:+ RuntimeError: If the request to fetch weather data fails.+ """+ import requests++ url = f"https://wttr.in/{location}?format=%C+%t"++ response = requests.get(url)+ if response.status_code == 200:+ weather_data = response.text+ return f"The weather in {location} is {weather_data}."+ else:+ raise RuntimeError(f"Failed to get weather data, status code: {response.status_code}")++ yield get_weather+++def print_tool_func():+ """Fixture to create a tool with default settings and clean up after the test."""++ def print_tool(message: str):+ """+ Args:+ message (str): The message to print.++ Returns:+ str: The message that was printed.+ """+ print(message)+ return message++ yield print_tool+++def roll_dice_tool_func():+ def roll_dice():+ """+ Rolls a 6 sided die.++ Returns:+ str: The roll result.+ """+ import time++ time.sleep(1)+ return "Rolled a 10!"++ yield roll_dice+++def dummy_beta_message_batch() -> BetaMessageBatch:+ return BetaMessageBatch(+ id="msgbatch_013Zva2CMHLNnXjNJJKqJ2EF",+ archived_at=datetime(2024, 8, 20, 18, 37, 24, 100435, tzinfo=timezone.utc),+ cancel_initiated_at=datetime(2024, 8, 20, 18, 37, 24, 100435, tzinfo=timezone.utc),+ created_at=datetime(2024, 8, 20, 18, 37, 24, 100435, tzinfo=timezone.utc),+ ended_at=datetime(2024, 8, 20, 18, 37, 24, 100435, tzinfo=timezone.utc),+ expires_at=datetime(2024, 8, 20, 18, 37, 24, 100435, tzinfo=timezone.utc),+ processing_status="in_progress",+ request_counts=BetaMessageBatchRequestCounts(+ canceled=10,+ errored=30,+ expired=10,+ processing=100,+ succeeded=50,+ ),+ results_url="https://api.anthropic.com/v1/messages/batches/msgbatch_013Zva2CMHLNnXjNJJKqJ2EF/results",+ type="message_batch",+ )+++# --- Model Sweep ---+# Global flag to track server state+_server_started = False+_server_url = None+++def _start_server_once() -> str:+ """Start server exactly once, return URL"""+ global _server_started, _server_url++ if _server_started and _server_url:+ return _server_url++ url = os.getenv("LETTA_SERVER_URL", "http://localhost:8283")++ # Check if already running+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:+ if s.connect_ex(("localhost", 8283)) == 0:+ _server_started = True+ _server_url = url+ return url++ # Start server (your existing logic)+ if not os.getenv("LETTA_SERVER_URL"):++ def _run_server():+ load_dotenv()+ from letta.server.rest_api.app import start_server++ start_server(debug=True)++ thread = threading.Thread(target=_run_server, daemon=True)+ thread.start()++ # Poll until up+ timeout_seconds = 30+ deadline = time.time() + timeout_seconds+ while time.time() < deadline:+ try:+ resp = requests.get(url + "/v1/health")+ if resp.status_code < 500:+ break+ except requests.exceptions.RequestException:+ pass+ time.sleep(0.1)+ else:+ raise RuntimeError(f"Could not reach {url} within {timeout_seconds}s")++ _server_started = True+ _server_url = url+ return url+++# ------------------------------+# Fixtures+# ------------------------------++[email protected](scope="module")+def server_url() -> str:+ """Return URL of already-started server"""+ return _start_server_once()++[email protected](scope="module")+def client(server_url: str) -> Letta:+ """+ Creates and returns a synchronous Letta REST client for testing.+ """+ client_instance = Letta(base_url=server_url)+ yield client_instance++[email protected](scope="function")+def async_client(server_url: str) -> AsyncLetta:+ """+ Creates and returns an asynchronous Letta REST client for testing.+ """+ async_client_instance = AsyncLetta(base_url=server_url)+ yield async_client_instance++[email protected](scope="module")+def agent_state(client: Letta) -> AgentState:+ """+ Creates and returns an agent state for testing with a pre-configured agent.+ The agent is named 'supervisor' and is configured with base tools and the roll_dice tool.+ """+ client.tools.upsert_base_tools()++ send_message_tool = client.tools.list(name="send_message")[0]+ agent_state_instance = client.agents.create(+ name="supervisor",+ include_base_tools=False,+ tool_ids=[send_message_tool.id],+ model="openai/gpt-4o",+ embedding="letta/letta-free",+ tags=["supervisor"],+ )+ yield agent_state_instance++ client.agents.delete(agent_state_instance.id)++[email protected](scope="module")+def all_available_llm_configs(client: Letta) -> [LLMConfig]:+ """+ Returns a list of all available LLM configs.+ """+ llm_configs = client.models.list()+ return llm_configs+++# create a client to the started server started at+def get_available_llm_configs() -> [LLMConfig]:+ """Get configs, starting server if needed"""+ server_url = _start_server_once()+ temp_client = Letta(base_url=server_url)+ return temp_client.models.list()+++# dynamically insert llm_config paramter at collection time+def pytest_generate_tests(metafunc):+ """Dynamically parametrize tests that need llm_config."""+ if "llm_config" in metafunc.fixturenames:+ configs = get_available_llm_configs()+ if configs:+ metafunc.parametrize("llm_config", configs, ids=[c.model for c in configs])diff --git a/.github/scripts/model-sweep/feature_mappings.json b/.github/scripts/model-sweep/feature_mappings.jsonnew file mode 100644index 000000000..41222e154--- /dev/null+++ b/.github/scripts/model-sweep/feature_mappings.json@@ -0,0 +1,21 @@+{+ "Basic": [+ "test_greeting_with_assistant_message",+ "test_greeting_without_assistant_message",+ "test_async_greeting_with_assistant_message",+ "test_agent_loop_error",+ "test_step_stream_agent_loop_error",+ "test_step_streaming_greeting_with_assistant_message",+ "test_step_streaming_greeting_without_assistant_message",+ "test_step_streaming_tool_call",+ "test_tool_call",+ "test_auto_summarize"+ ],+ "Token Streaming": [+ "test_token_streaming_greeting_with_assistant_message",+ "test_token_streaming_greeting_without_assistant_message",+ "test_token_streaming_agent_loop_error",+ "test_token_streaming_tool_call"+ ],+ "Multimodal": ["test_base64_image_input", "test_url_image_input"]+}diff --git a/.github/scripts/model-sweep/generate_model_sweep_markdown.py b/.github/scripts/model-sweep/generate_model_sweep_markdown.pynew file mode 100644index 000000000..38552a8c1--- /dev/null+++ b/.github/scripts/model-sweep/generate_model_sweep_markdown.py@@ -0,0 +1,495 @@+#!/usr/bin/env python3+import json+import os+import sys+from collections import defaultdict+from datetime import datetime+++def load_feature_mappings(config_file=None):+ """Load feature mappings from config file."""+ if config_file is None:+ # Default to feature_mappings.json in the same directory as this script+ script_dir = os.path.dirname(os.path.abspath(__file__))+ config_file = os.path.join(script_dir, "feature_mappings.json")++ try:+ with open(config_file, "r") as f:+ return json.load(f)+ except FileNotFoundError:+ print(f"Error: Could not find feature mappings config file '{config_file}'")+ sys.exit(1)+ except json.JSONDecodeError:+ print(f"Error: Invalid JSON in feature mappings config file '{config_file}'")+ sys.exit(1)+++def get_support_status(passed_tests, feature_tests):+ """Determine support status for a feature category."""+ if not feature_tests:+ return "❓" # Unknown - no tests for this feature++ # Filter out error tests when checking for support+ non_error_tests = [test for test in feature_tests if not test.endswith("_error")]+ error_tests = [test for test in feature_tests if test.endswith("_error")]++ # Check which non-error tests passed+ passed_non_error_tests = [test for test in non_error_tests if test in passed_tests]++ # If there are no non-error tests, only error tests, treat as unknown+ if not non_error_tests:+ return "❓" # Only error tests available++ # Support is based only on non-error tests+ if len(passed_non_error_tests) == len(non_error_tests):+ return "✅" # Full support+ elif len(passed_non_error_tests) == 0:+ return "❌" # No support+ else:+ return "⚠️" # Partial support+++def categorize_tests(all_test_names, feature_mapping):+ """Categorize test names into feature buckets."""+ categorized = {feature: [] for feature in feature_mapping.keys()}++ for test_name in all_test_names:+ for feature, test_patterns in feature_mapping.items():+ if test_name in test_patterns:+ categorized[feature].append(test_name)+ break++ return categorized+++def calculate_support_score(feature_support, feature_order):+ """Calculate a numeric support score for ranking models.++ For partial support, the score is weighted by the position of the feature+ in the feature_order list (earlier features get higher weight).+ """+ score = 0+ max_features = len(feature_order)++ for feature, status in feature_support.items():+ # Get position weight (earlier features get higher weight)+ if feature in feature_order:+ position_weight = (max_features - feature_order.index(feature)) / max_features+ else:+ position_weight = 0.5 # Default weight for unmapped features++ if status == "✅": # Full support+ score += 10 * position_weight+ elif status == "⚠️": # Partial support - weighted by column position+ score += 5 * position_weight+ elif status == "❌": # No support+ score += 1 * position_weight+ # Unknown (❓) gets 0 points+ return score+++def calculate_provider_support_score(models_data, feature_order):+ """Calculate a provider-level support score based on all models' support scores."""+ if not models_data:+ return 0++ # Calculate the average support score across all models in the provider+ total_score = sum(model["support_score"] for model in models_data)+ return total_score / len(models_data)+++def get_test_function_line_numbers(test_file_path):+ """Extract line numbers for test functions from the test file."""+ test_line_numbers = {}++ try:+ with open(test_file_path, "r") as f:+ lines = f.readlines()++ for i, line in enumerate(lines, 1):+ if "def test_" in line and line.strip().startswith("def test_"):+ # Extract function name+ func_name = line.strip().split("def ")[1].split("(")[0]+ test_line_numbers[func_name] = i+ except FileNotFoundError:+ print(f"Warning: Could not find test file at {test_file_path}")++ return test_line_numbers+++def get_github_repo_info():+ """Get GitHub repository information from git remote."""+ try:+ # Try to get the GitHub repo URL from git remote+ import subprocess++ result = subprocess.run(["git", "remote", "get-url", "origin"], capture_output=True, text=True, cwd=os.path.dirname(__file__))+ if result.returncode == 0:+ remote_url = result.stdout.strip()+ # Parse GitHub URL+ if "github.com" in remote_url:+ if remote_url.startswith("https://"):+ # https://github.com/user/repo.git -> user/repo+ repo_path = remote_url.replace("https://github.com/", "").replace(".git", "")+ elif remote_url.startswith("git@"):+ # [email protected]:user/repo.git -> user/repo+ repo_path = remote_url.split(":")[1].replace(".git", "")+ else:+ return None+ return repo_path+ except:+ pass++ # Default fallback+ return "letta-ai/letta"+++def generate_test_details(model_info, feature_mapping):+ """Generate detailed test results for a model."""+ details = []++ # Get test function line numbers+ script_dir = os.path.dirname(os.path.abspath(__file__))+ test_file_path = os.path.join(script_dir, "model_sweep.py")+ test_line_numbers = get_test_function_line_numbers(test_file_path)++ # Use the main branch GitHub URL+ base_github_url = "https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py"++ for feature, tests in model_info["categorized_tests"].items():+ if not tests:+ continue++ details.append(f"### {feature}")+ details.append("")++ for test in sorted(tests):+ if test in model_info["passed_tests"]:+ status = "✅"+ elif test in model_info["failed_tests"]:+ status = "❌"+ else:+ status = "❓"++ # Create GitHub link if we have line number info+ if test in test_line_numbers:+ line_num = test_line_numbers[test]+ github_link = f"{base_github_url}#L{line_num}"+ details.append(f"- {status} [`{test}`]({github_link})")+ else:+ details.append(f"- {status} `{test}`")+ details.append("")++ return details+++def calculate_column_widths(all_provider_data, feature_mapping):+ """Calculate the maximum width needed for each column across all providers."""+ widths = {"model": len("Model"), "context_window": len("Context Window"), "last_scanned": len("Last Scanned")}++ # Feature column widths+ for feature in feature_mapping.keys():+ widths[feature] = len(feature)++ # Check all model data for maximum widths+ for provider_data in all_provider_data.values():+ for model_info in provider_data:+ # Model name width (including backticks)+ model_width = len(f"`{model_info['name']}`")+ widths["model"] = max(widths["model"], model_width)++ # Context window width (with commas)+ context_width = len(f"{model_info['context_window']:,}")+ widths["context_window"] = max(widths["context_window"], context_width)++ # Last scanned width+ widths["last_scanned"] = max(widths["last_scanned"], len(str(model_info["last_scanned"])))++ # Feature support symbols are always 2 chars, so no need to check++ return widths+++def process_model_sweep_report(input_file, output_file, config_file=None, debug=False):+ """Convert model sweep JSON data to MDX report."""++ # Load feature mappings from config file+ feature_mapping = load_feature_mappings(config_file)++ # if debug:+ # print("DEBUG: Feature mappings loaded:")+ # for feature, tests in feature_mapping.items():+ # print(f" {feature}: {tests}")+ # print()++ # Read the JSON data+ with open(input_file, "r") as f:+ data = json.load(f)++ tests = data.get("tests", [])++ # if debug:+ # print("DEBUG: Tests loaded:")+ # print([test['outcome'] for test in tests if 'haiku' in test['nodeid']])++ # Calculate summary statistics+ providers = set(test["metadata"]["llm_config"]["provider_name"] for test in tests)+ models = set(test["metadata"]["llm_config"]["model"] for test in tests)+ total_tests = len(tests)++ # Start building the MDX+ mdx_lines = [+ "---",+ "title: Support Models",+ f"generated: {datetime.now().isoformat()}",+ "---",+ "",+ "# Supported Models",+ "",+ "## Overview",+ "",+ "Letta routinely runs automated scans against available providers and models. These are the results of the latest scan.",+ "",+ f"Ran {total_tests} tests against {len(models)} models across {len(providers)} providers on {datetime.now().strftime('%B %dth, %Y')}",+ "",+ "",+ ]++ # Group tests by provider+ provider_groups = defaultdict(list)+ for test in tests:+ provider_name = test["metadata"]["llm_config"]["provider_name"]+ provider_groups[provider_name].append(test)++ # Process all providers first to collect model data+ all_provider_data = {}+ provider_support_scores = {}++ for provider_name in provider_groups.keys():+ provider_tests = provider_groups[provider_name]++ # Group tests by model within this provider+ model_groups = defaultdict(list)+ for test in provider_tests:+ model_name = test["metadata"]["llm_config"]["model"]+ model_groups[model_name].append(test)++ # Process all models to calculate support scores for ranking+ model_data = []+ for model_name in model_groups.keys():+ model_tests = model_groups[model_name]++ # if debug:+ # print(f"DEBUG: Processing model '{model_name}' in provider '{provider_name}'")++ # Extract unique test names for passed and failed tests+ passed_tests = set()+ failed_tests = set()+ all_test_names = set()++ for test in model_tests:+ # Extract test name from nodeid (split on :: and [)+ test_name = test["nodeid"].split("::")[1].split("[")[0]+ all_test_names.add(test_name)++ # if debug:+ # print(f" Test name: {test_name}")+ # print(f" Outcome: {test}")+ if test["outcome"] == "passed":+ passed_tests.add(test_name)+ elif test["outcome"] == "failed":+ failed_tests.add(test_name)++ # if debug:+ # print(f" All test names found: {sorted(all_test_names)}")+ # print(f" Passed tests: {sorted(passed_tests)}")+ # print(f" Failed tests: {sorted(failed_tests)}")++ # Categorize tests into features+ categorized_tests = categorize_tests(all_test_names, feature_mapping)++ # if debug:+ # print(f" Categorized tests:")+ # for feature, tests in categorized_tests.items():+ # print(f" {feature}: {tests}")++ # Determine support status for each feature+ feature_support = {}+ for feature_name in feature_mapping.keys():+ feature_support[feature_name] = get_support_status(passed_tests, categorized_tests[feature_name])++ # if debug:+ # print(f" Feature support:")+ # for feature, status in feature_support.items():+ # print(f" {feature}: {status}")+ # print()++ # Get context window and last scanned time+ context_window = model_tests[0]["metadata"]["llm_config"]["context_window"]++ # Try to get time_last_scanned from metadata, fallback to current time+ try:+ last_scanned = model_tests[0]["metadata"].get(+ "time_last_scanned", model_tests[0]["metadata"].get("timestamp", datetime.now().isoformat())+ )+ # Format timestamp if it's a full ISO string+ if "T" in str(last_scanned):+ last_scanned = str(last_scanned).split("T")[0] # Just the date part+ except:+ last_scanned = "Unknown"++ # Calculate support score for ranking+ feature_order = list(feature_mapping.keys())+ support_score = calculate_support_score(feature_support, feature_order)++ # Store model data for sorting+ model_data.append(+ {+ "name": model_name,+ "feature_support": feature_support,+ "context_window": context_window,+ "last_scanned": last_scanned,+ "support_score": support_score,+ "failed_tests": failed_tests,+ "passed_tests": passed_tests,+ "categorized_tests": categorized_tests,+ }+ )++ # Sort models by support score (descending) then by name (ascending)+ model_data.sort(key=lambda x: (-x["support_score"], x["name"]))++ # Store provider data+ all_provider_data[provider_name] = model_data+ provider_support_scores[provider_name] = calculate_provider_support_score(model_data, list(feature_mapping.keys()))++ # Calculate column widths for consistent formatting (add details column)+ column_widths = calculate_column_widths(all_provider_data, feature_mapping)+ column_widths["details"] = len("Details")++ # Sort providers by support score (descending) then by name (ascending)+ sorted_providers = sorted(provider_support_scores.keys(), key=lambda x: (-provider_support_scores[x], x))++ # Generate tables for all providers first+ for provider_name in sorted_providers:+ model_data = all_provider_data[provider_name]+ support_score = provider_support_scores[provider_name]++ # Create dynamic headers with proper padding and centering+ feature_names = list(feature_mapping.keys())++ # Build header row with left-aligned first column, centered others+ header_parts = [f"{'Model':<{column_widths['model']}}"]+ for feature in feature_names:+ header_parts.append(f"{feature:^{column_widths[feature]}}")+ header_parts.extend(+ [+ f"{'Context Window':^{column_widths['context_window']}}",+ f"{'Last Scanned':^{column_widths['last_scanned']}}",+ f"{'Details':^{column_widths['details']}}",+ ]+ )+ header_row = "| " + " | ".join(header_parts) + " |"++ # Build separator row with left-aligned first column, centered others+ separator_parts = [f"{'-' * column_widths['model']}"]+ for feature in feature_names:+ separator_parts.append(f":{'-' * (column_widths[feature] - 2)}:")+ separator_parts.extend(+ [+ f":{'-' * (column_widths['context_window'] - 2)}:",+ f":{'-' * (column_widths['last_scanned'] - 2)}:",+ f":{'-' * (column_widths['details'] - 2)}:",+ ]+ )+ separator_row = "|" + "|".join(separator_parts) + "|"++ # Add provider section without percentage+ mdx_lines.extend([f"## {provider_name}", "", header_row, separator_row])++ # Generate table rows for sorted models with proper padding+ for model_info in model_data:+ # Create anchor for model details+ model_anchor = model_info["name"].replace("/", "_").replace(":", "_").replace("-", "_").lower()+ details_anchor = f"{provider_name.lower().replace(' ', '_')}_{model_anchor}_details"++ # Build row with left-aligned first column, centered others+ row_parts = [f"`{model_info['name']}`".ljust(column_widths["model"])]+ for feature in feature_names:+ row_parts.append(f"{model_info['feature_support'][feature]:^{column_widths[feature]}}")+ row_parts.extend(+ [+ f"{model_info['context_window']:,}".center(column_widths["context_window"]),+ f"{model_info['last_scanned']}".center(column_widths["last_scanned"]),+ f"[View](#{details_anchor})".center(column_widths["details"]),+ ]+ )+ row = "| " + " | ".join(row_parts) + " |"+ mdx_lines.append(row)++ # Add spacing between provider tables+ mdx_lines.extend(["", ""])++ # Add detailed test results section after all tables+ mdx_lines.extend(["---", "", "# Detailed Test Results", ""])++ for provider_name in sorted_providers:+ model_data = all_provider_data[provider_name]+ mdx_lines.extend([f"## {provider_name}", ""])++ for model_info in model_data:+ model_anchor = model_info["name"].replace("/", "_").replace(":", "_").replace("-", "_").lower()+ details_anchor = f"{provider_name.lower().replace(' ', '_')}_{model_anchor}_details"+ mdx_lines.append(f'<a id="{details_anchor}"></a>')+ mdx_lines.append(f"### {model_info['name']}")+ mdx_lines.append("")++ # Add test details+ test_details = generate_test_details(model_info, feature_mapping)+ mdx_lines.extend(test_details)++ # Add spacing between providers in details section+ mdx_lines.extend(["", ""])++ # Write the MDX file+ with open(output_file, "w") as f:+ f.write("\n".join(mdx_lines))++ print(f"Model sweep report saved to {output_file}")+++def main():+ input_file = "model_sweep_report.json"+ output_file = "model_sweep_report.mdx"+ config_file = None+ debug = False++ # Allow command line arguments+ if len(sys.argv) > 1:+ # Use the file located in the same directory as this script+ script_dir = os.path.dirname(os.path.abspath(__file__))+ input_file = os.path.join(script_dir, sys.argv[1])+ if len(sys.argv) > 2:+ # Use the file located in the same directory as this script+ script_dir = os.path.dirname(os.path.abspath(__file__))+ output_file = os.path.join(script_dir, sys.argv[2])+ if len(sys.argv) > 3:+ config_file = sys.argv[3]+ if len(sys.argv) > 4 and sys.argv[4] == "--debug":+ debug = True++ try:+ process_model_sweep_report(input_file, output_file, config_file, debug)+ except FileNotFoundError:+ print(f"Error: Could not find input file '{input_file}'")+ sys.exit(1)+ except json.JSONDecodeError:+ print(f"Error: Invalid JSON in file '{input_file}'")+ sys.exit(1)+ except Exception as e:+ print(f"Error: {e}")+ sys.exit(1)+++if __name__ == "__main__":+ main()diff --git a/.github/scripts/model-sweep/model_sweep.py b/.github/scripts/model-sweep/model_sweep.pynew file mode 100644index 000000000..c92b93643--- /dev/null+++ b/.github/scripts/model-sweep/model_sweep.py@@ -0,0 +1,786 @@+import base64+import json+import os+import socket+import threading+import time+import uuid+from typing import Any, Dict, List++import httpx+import pytest+import requests+from dotenv import load_dotenv+from letta_client import Letta, MessageCreate, Run+from letta_client.core.api_error import ApiError+from letta_client.types import (+ AssistantMessage,+ Base64Image,+ ImageContent,+ LettaUsageStatistics,+ ReasoningMessage,+ TextContent,+ ToolCallMessage,+ ToolReturnMessage,+ UrlImage,+ UserMessage,+)++from letta.schemas.agent import AgentState+from letta.schemas.llm_config import LLMConfig++# ------------------------------+# Helper Functions and Constants+# ------------------------------+++def get_llm_config(filename: str, llm_config_dir: str = "tests/configs/llm_model_configs") -> LLMConfig:+ filename = os.path.join(llm_config_dir, filename)+ config_data = json.load(open(filename, "r"))+ llm_config = LLMConfig(**config_data)+ return llm_config+++def roll_dice(num_sides: int) -> int:+ """+ Returns a random number between 1 and num_sides.+ Args:+ num_sides (int): The number of sides on the die.+ Returns:+ int: A random integer between 1 and num_sides, representing the die roll.+ """+ import random++ return random.randint(1, num_sides)+++USER_MESSAGE_OTID = str(uuid.uuid4())+USER_MESSAGE_RESPONSE: str = "Teamwork makes the dream work"+USER_MESSAGE_FORCE_REPLY: List[MessageCreate] = [+ MessageCreate(+ role="user",+ content=f"This is an automated test message. Call the send_message tool with the message '{USER_MESSAGE_RESPONSE}'.",+ otid=USER_MESSAGE_OTID,+ )+]+USER_MESSAGE_ROLL_DICE: List[MessageCreate] = [+ MessageCreate(+ role="user",+ content="This is an automated test message. Call the roll_dice tool with 16 sides and tell me the outcome.",+ otid=USER_MESSAGE_OTID,+ )+]+URL_IMAGE = "https://upload.wikimedia.org/wikipedia/commons/a/a7/Camponotus_flavomarginatus_ant.jpg"+USER_MESSAGE_URL_IMAGE: List[MessageCreate] = [+ MessageCreate(+ role="user",+ content=[+ ImageContent(source=UrlImage(url=URL_IMAGE)),+ TextContent(text="What is in this image?"),+ ],+ otid=USER_MESSAGE_OTID,+ )+]+BASE64_IMAGE = base64.standard_b64encode(httpx.get(URL_IMAGE).content).decode("utf-8")+USER_MESSAGE_BASE64_IMAGE: List[MessageCreate] = [+ MessageCreate(+ role="user",+ content=[+ ImageContent(source=Base64Image(data=BASE64_IMAGE, media_type="image/jpeg")),+ TextContent(text="What is in this image?"),+ ],+ otid=USER_MESSAGE_OTID,+ )+]+all_configs = [+ "openai-gpt-4o-mini.json",+ # "azure-gpt-4o-mini.json", # TODO: Re-enable on new agent loop+ "claude-3-5-sonnet.json",+ "claude-3-7-sonnet.json",+ "claude-3-7-sonnet-extended.json",+ "gemini-1.5-pro.json",+ "gemini-2.5-flash-vertex.json",+ "gemini-2.5-pro-vertex.json",+ "together-qwen-2.5-72b-instruct.json",+ "ollama.json",+]+requested = os.getenv("LLM_CONFIG_FILE")+filenames = [requested] if requested else all_configs+TESTED_LLM_CONFIGS: List[LLMConfig] = [get_llm_config(fn) for fn in filenames]+++def assert_greeting_with_assistant_message_response(+ messages: List[Any],+ streaming: bool = False,+ token_streaming: bool = False,+ from_db: bool = False,+) -> None:+ """+ Asserts that the messages list follows the expected sequence:+ ReasoningMessage -> AssistantMessage.+ """+ expected_message_count = 3 if streaming or from_db else 2+ assert len(messages) == expected_message_count++ index = 0+ if from_db:+ assert isinstance(messages[index], UserMessage)+ assert messages[index].otid == USER_MESSAGE_OTID+ index += 1++ # Agent Step 1+ assert isinstance(messages[index], ReasoningMessage)+ assert messages[index].otid and messages[index].otid[-1] == "0"+ index += 1++ assert isinstance(messages[index], AssistantMessage)+ if not token_streaming:+ assert USER_MESSAGE_RESPONSE in messages[index].content+ assert messages[index].otid and messages[index].otid[-1] == "1"+ index += 1++ if streaming:+ assert isinstance(messages[index], LettaUsageStatistics)+ assert messages[index].prompt_tokens > 0+ assert messages[index].completion_tokens > 0+ assert messages[index].total_tokens > 0+ assert messages[index].step_count > 0+++def assert_greeting_without_assistant_message_response(+ messages: List[Any],+ streaming: bool = False,+ token_streaming: bool = False,+ from_db: bool = False,+) -> None:+ """+ Asserts that the messages list follows the expected sequence:+ ReasoningMessage -> ToolCallMessage -> ToolReturnMessage.+ """+ expected_message_count = 4 if streaming or from_db else 3+ assert len(messages) == expected_message_count++ index = 0+ if from_db:+ assert isinstance(messages[index], UserMessage)+ assert messages[index].otid == USER_MESSAGE_OTID+ index += 1++ # Agent Step 1+ assert isinstance(messages[index], ReasoningMessage)+ assert messages[index].otid and messages[index].otid[-1] == "0"+ index += 1++ assert isinstance(messages[index], ToolCallMessage)+ assert messages[index].tool_call.name == "send_message"+ if not token_streaming:+ assert USER_MESSAGE_RESPONSE in messages[index].tool_call.arguments+ assert messages[index].otid and messages[index].otid[-1] == "1"+ index += 1++ # Agent Step 2+ assert isinstance(messages[index], ToolReturnMessage)+ assert messages[index].otid and messages[index].otid[-1] == "0"+ index += 1++ if streaming:+ assert isinstance(messages[index], LettaUsageStatistics)+++def assert_tool_call_response(+ messages: List[Any],+ streaming: bool = False,+ from_db: bool = False,+) -> None:+ """+ Asserts that the messages list follows the expected sequence:+ ReasoningMessage -> ToolCallMessage -> ToolReturnMessage ->+ ReasoningMessage -> AssistantMessage.+ """+ expected_message_count = 6 if streaming else 7 if from_db else 5+ assert len(messages) == expected_message_count++ index = 0+ if from_db:+ assert isinstance(messages[index], UserMessage)+ assert messages[index].otid == USER_MESSAGE_OTID+ index += 1++ # Agent Step 1+ assert isinstance(messages[index], ReasoningMessage)+ assert messages[index].otid and messages[index].otid[-1] == "0"+ index += 1++ assert isinstance(messages[index], ToolCallMessage)+ assert messages[index].otid and messages[index].otid[-1] == "1"+ index += 1++ # Agent Step 2+ assert isinstance(messages[index], ToolReturnMessage)+ assert messages[index].otid and messages[index].otid[-1] == "0"+ index += 1++ # Hidden User Message+ if from_db:+ assert isinstance(messages[index], UserMessage)+ assert "request_heartbeat=true" in messages[index].content+ index += 1++ # Agent Step 3+ assert isinstance(messages[index], ReasoningMessage)+ assert messages[index].otid and messages[index].otid[-1] == "0"+ index += 1++ assert isinstance(messages[index], AssistantMessage)+ assert messages[index].otid and messages[index].otid[-1] == "1"+ index += 1++ if streaming:+ assert isinstance(messages[index], LettaUsageStatistics)+++def assert_image_input_response(+ messages: List[Any],+ streaming: bool = False,+ token_streaming: bool = False,+ from_db: bool = False,+) -> None:+ """+ Asserts that the messages list follows the expected sequence:+ ReasoningMessage -> AssistantMessage.+ """+ expected_message_count = 3 if streaming or from_db else 2+ assert len(messages) == expected_message_count++ index = 0+ if from_db:+ assert isinstance(messages[index], UserMessage)+ assert messages[index].otid == USER_MESSAGE_OTID+ index += 1++ # Agent Step 1+ assert isinstance(messages[index], ReasoningMessage)+ assert messages[index].otid and messages[index].otid[-1] == "0"+ index += 1++ assert isinstance(messages[index], AssistantMessage)+ assert messages[index].otid and messages[index].otid[-1] == "1"+ index += 1++ if streaming:+ assert isinstance(messages[index], LettaUsageStatistics)+ assert messages[index].prompt_tokens > 0+ assert messages[index].completion_tokens > 0+ assert messages[index].total_tokens > 0+ assert messages[index].step_count > 0+++def accumulate_chunks(chunks: List[Any]) -> List[Any]:+ """+ Accumulates chunks into a list of messages.+ """+ messages = []+ current_message = None+ prev_message_type = None+ for chunk in chunks:+ current_message_type = chunk.message_type+ if prev_message_type != current_message_type:+ messages.append(current_message)+ current_message = None+ if current_message is None:+ current_message = chunk+ else:+ pass # TODO: actually accumulate the chunks. For now we only care about the count+ prev_message_type = current_message_type+ messages.append(current_message)+ return [m for m in messages if m is not None]+++def wait_for_run_completion(client: Letta, run_id: str, timeout: float = 30.0, interval: float = 0.5) -> Run:+ start = time.time()+ while True:+ run = client.runs.retrieve(run_id)+ if run.status == "completed":+ return run+ if run.status == "failed":+ raise RuntimeError(f"Run {run_id} did not complete: status = {run.status}")+ if time.time() - start > timeout:+ raise TimeoutError(f"Run {run_id} did not complete within {timeout} seconds (last status: {run.status})")+ time.sleep(interval)+++def assert_tool_response_dict_messages(messages: List[Dict[str, Any]]) -> None:+ """+ Asserts that a list of message dictionaries contains the expected types and statuses.++ Expected order:+ 1. reasoning_message+ 2. tool_call_message+ 3. tool_return_message (with status 'success')+ 4. reasoning_message+ 5. assistant_message+ """+ assert isinstance(messages, list)+ assert messages[0]["message_type"] == "reasoning_message"+ assert messages[1]["message_type"] == "assistant_message"+++# ------------------------------+# Test Cases+# ------------------------------++# def test_that_ci_workflow_works(+# disable_e2b_api_key: Any,+# client: Letta,+# agent_state: AgentState,+# llm_config: LLMConfig,+# json_metadata: pytest.FixtureRequest,+# ) -> None:+# """+# Tests that the CI workflow works.+# """+# json_metadata["test_type"] = "debug"+++def test_greeting_with_assistant_message(+ disable_e2b_api_key: Any,+ client: Letta,+ agent_state: AgentState,+ llm_config: LLMConfig,+ json_metadata: pytest.FixtureRequest,+) -> None:+ """+ Tests sending a message with a synchronous client.+ Verifies that the response messages follow the expected order.+ """+ json_metadata["llm_config"] = dict(llm_config)+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_FORCE_REPLY,+ )+ assert_greeting_with_assistant_message_response(response.messages)+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)+ assert_greeting_with_assistant_message_response(messages_from_db, from_db=True)+++def test_greeting_without_assistant_message(+ disable_e2b_api_key: Any,+ client: Letta,+ llm_config: LLMConfig,+ agent_state: AgentState,+ json_metadata: pytest.FixtureRequest,+) -> None:+ """+ Tests sending a message with a synchronous client.+ Verifies that the response messages follow the expected order.+ """+ json_metadata["llm_config"] = dict(llm_config)+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_FORCE_REPLY,+ use_assistant_message=False,+ )+ assert_greeting_without_assistant_message_response(response.messages)+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id, use_assistant_message=False)+ assert_greeting_without_assistant_message_response(messages_from_db, from_db=True)+++def test_tool_call(+ disable_e2b_api_key: Any,+ client: Letta,+ llm_config: LLMConfig,+ agent_state: AgentState,+ json_metadata: pytest.FixtureRequest,+) -> None:+ """+ Tests sending a message with a synchronous client.+ Verifies that the response messages follow the expected order.+ """+ json_metadata["llm_config"] = dict(llm_config)+ dice_tool = client.tools.upsert_from_function(func=roll_dice)+ client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_ROLL_DICE,+ )+ assert_tool_call_response(response.messages)+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)+ assert_tool_call_response(messages_from_db, from_db=True)+++def test_url_image_input(+ disable_e2b_api_key: Any,+ client: Letta,+ llm_config: LLMConfig,+ agent_state: AgentState,+ json_metadata: pytest.FixtureRequest,+) -> None:+ """+ Tests sending a message with a synchronous client.+ Verifies that the response messages follow the expected order.+ """+ json_metadata["llm_config"] = dict(llm_config)+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_URL_IMAGE,+ )+ assert_image_input_response(response.messages)+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)+ assert_image_input_response(messages_from_db, from_db=True)+++def test_base64_image_input(+ disable_e2b_api_key: Any,+ client: Letta,+ llm_config: LLMConfig,+ agent_state: AgentState,+ json_metadata: pytest.FixtureRequest,+) -> None:+ """+ Tests sending a message with a synchronous client.+ Verifies that the response messages follow the expected order.+ """+ json_metadata["llm_config"] = dict(llm_config)+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_BASE64_IMAGE,+ )+ assert_image_input_response(response.messages)+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)+ assert_image_input_response(messages_from_db, from_db=True)+++def test_agent_loop_error(+ disable_e2b_api_key: Any,+ client: Letta,+ llm_config: LLMConfig,+ agent_state: AgentState,+ json_metadata: pytest.FixtureRequest,+) -> None:+ """+ Tests sending a message with a synchronous client.+ Verifies that no new messages are persisted on error.+ """+ json_metadata["llm_config"] = dict(llm_config)+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)+ tools = agent_state.tools+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config, tool_ids=[])+ with pytest.raises(ApiError):+ client.agents.messages.create(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_FORCE_REPLY,+ )+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)+ assert len(messages_from_db) == 0+ client.agents.modify(agent_id=agent_state.id, tool_ids=[t.id for t in tools])+++def test_step_streaming_greeting_with_assistant_message(+ disable_e2b_api_key: Any,+ client: Letta,+ llm_config: LLMConfig,+ agent_state: AgentState,+ json_metadata: pytest.FixtureRequest,+) -> None:+ """+ Tests sending a streaming message with a synchronous client.+ Checks that each chunk in the stream has the correct message types.+ """+ json_metadata["llm_config"] = dict(llm_config)+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_FORCE_REPLY,+ )+ chunks = list(response)+ messages = accumulate_chunks(chunks)+ assert_greeting_with_assistant_message_response(messages, streaming=True)+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)+ assert_greeting_with_assistant_message_response(messages_from_db, from_db=True)+++def test_step_streaming_greeting_without_assistant_message(+ disable_e2b_api_key: Any,+ client: Letta,+ llm_config: LLMConfig,+ agent_state: AgentState,+ json_metadata: pytest.FixtureRequest,+) -> None:+ """+ Tests sending a streaming message with a synchronous client.+ Checks that each chunk in the stream has the correct message types.+ """+ json_metadata["llm_config"] = dict(llm_config)+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_FORCE_REPLY,+ use_assistant_message=False,+ )+ chunks = list(response)+ messages = accumulate_chunks(chunks)+ assert_greeting_without_assistant_message_response(messages, streaming=True)+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id, use_assistant_message=False)+ assert_greeting_without_assistant_message_response(messages_from_db, from_db=True)+++def test_step_streaming_tool_call(+ disable_e2b_api_key: Any,+ client: Letta,+ llm_config: LLMConfig,+ agent_state: AgentState,+ json_metadata: pytest.FixtureRequest,+) -> None:+ """+ Tests sending a streaming message with a synchronous client.+ Checks that each chunk in the stream has the correct message types.+ """+ json_metadata["llm_config"] = dict(llm_config)+ dice_tool = client.tools.upsert_from_function(func=roll_dice)+ agent_state = client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_ROLL_DICE,+ )+ chunks = list(response)+ messages = accumulate_chunks(chunks)+ assert_tool_call_response(messages, streaming=True)+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)+ assert_tool_call_response(messages_from_db, from_db=True)+++def test_step_stream_agent_loop_error(+ disable_e2b_api_key: Any,+ client: Letta,+ llm_config: LLMConfig,+ agent_state: AgentState,+ json_metadata: pytest.FixtureRequest,+) -> None:+ """+ Tests sending a message with a synchronous client.+ Verifies that no new messages are persisted on error.+ """+ json_metadata["llm_config"] = dict(llm_config)+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)+ tools = agent_state.tools+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config, tool_ids=[])+ with pytest.raises(ApiError):+ response = client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_FORCE_REPLY,+ )+ list(response)++ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)+ assert len(messages_from_db) == 0+ client.agents.modify(agent_id=agent_state.id, tool_ids=[t.id for t in tools])+++def test_token_streaming_greeting_with_assistant_message(+ disable_e2b_api_key: Any,+ client: Letta,+ llm_config: LLMConfig,+ agent_state: AgentState,+ json_metadata: pytest.FixtureRequest,+) -> None:+ """+ Tests sending a streaming message with a synchronous client.+ Checks that each chunk in the stream has the correct message types.+ """+ json_metadata["llm_config"] = dict(llm_config)+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_FORCE_REPLY,+ stream_tokens=True,+ )+ chunks = list(response)+ messages = accumulate_chunks(chunks)+ assert_greeting_with_assistant_message_response(messages, streaming=True, token_streaming=True)+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)+ assert_greeting_with_assistant_message_response(messages_from_db, from_db=True)+++def test_token_streaming_greeting_without_assistant_message(+ disable_e2b_api_key: Any,+ client: Letta,+ llm_config: LLMConfig,+ agent_state: AgentState,+ json_metadata: pytest.FixtureRequest,+) -> None:+ """+ Tests sending a streaming message with a synchronous client.+ Checks that each chunk in the stream has the correct message types.+ """+ json_metadata["llm_config"] = dict(llm_config)+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_FORCE_REPLY,+ use_assistant_message=False,+ stream_tokens=True,+ )+ chunks = list(response)+ messages = accumulate_chunks(chunks)+ assert_greeting_without_assistant_message_response(messages, streaming=True, token_streaming=True)+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id, use_assistant_message=False)+ assert_greeting_without_assistant_message_response(messages_from_db, from_db=True)+++def test_token_streaming_tool_call(+ disable_e2b_api_key: Any,+ client: Letta,+ llm_config: LLMConfig,+ agent_state: AgentState,+ json_metadata: pytest.FixtureRequest,+) -> None:+ """+ Tests sending a streaming message with a synchronous client.+ Checks that each chunk in the stream has the correct message types.+ """+ json_metadata["llm_config"] = dict(llm_config)+ dice_tool = client.tools.upsert_from_function(func=roll_dice)+ agent_state = client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_ROLL_DICE,+ stream_tokens=True,+ )+ chunks = list(response)+ messages = accumulate_chunks(chunks)+ assert_tool_call_response(messages, streaming=True)+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)+ assert_tool_call_response(messages_from_db, from_db=True)+++def test_token_streaming_agent_loop_error(+ disable_e2b_api_key: Any,+ client: Letta,+ llm_config: LLMConfig,+ agent_state: AgentState,+ json_metadata: pytest.FixtureRequest,+) -> None:+ """+ Tests sending a message with a synchronous client.+ Verifies that no new messages are persisted on error.+ """+ json_metadata["llm_config"] = dict(llm_config)+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)+ tools = agent_state.tools+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config, tool_ids=[])+ try:+ response = client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_FORCE_REPLY,+ stream_tokens=True,+ )+ list(response)+ except:+ pass # only some models throw an error TODO: make this consistent++ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)+ assert len(messages_from_db) == 0+ client.agents.modify(agent_id=agent_state.id, tool_ids=[t.id for t in tools])+++def test_async_greeting_with_assistant_message(+ disable_e2b_api_key: Any,+ client: Letta,+ llm_config: LLMConfig,+ agent_state: AgentState,+ json_metadata: pytest.FixtureRequest,+) -> None:+ """+ Tests sending a message as an asynchronous job using the synchronous client.+ Waits for job completion and asserts that the result messages are as expected.+ """+ json_metadata["llm_config"] = dict(llm_config)+ client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)++ run = client.agents.messages.create_async(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_FORCE_REPLY,+ )+ run = wait_for_run_completion(client, run.id)++ result = run.metadata.get("result")+ assert result is not None, "Run metadata missing 'result' key"++ messages = result["messages"]+ assert_tool_response_dict_messages(messages)+++def test_auto_summarize(+ disable_e2b_api_key: Any,+ client: Letta,+ llm_config: LLMConfig,+ json_metadata: pytest.FixtureRequest,+) -> None:+ """Test that summarization is automatically triggered."""+ json_metadata["llm_config"] = dict(llm_config)++ # pydantic prevents us for overriding the context window paramter in the passed LLMConfig+ new_llm_config = llm_config.model_dump()+ new_llm_config["context_window"] = 3000+ pinned_context_window_llm_config = LLMConfig(**new_llm_config)++ send_message_tool = client.tools.list(name="send_message")[0]+ temp_agent_state = client.agents.create(+ include_base_tools=False,+ tool_ids=[send_message_tool.id],+ llm_config=pinned_context_window_llm_config,+ embedding="letta/letta-free",+ tags=["supervisor"],+ )++ philosophical_question = """+You know, sometimes I wonder if the entire structure of our lives is built on a series of unexamined assumptions we just silently agreed to somewhere along the way—like how we all just decided that five days a week of work and two days of “rest” constitutes balance, or how 9-to-5 became the default rhythm of a meaningful life, or even how the idea of “success” got boiled down to job titles and property ownership and productivity metrics on a LinkedIn profile, when maybe none of that is actually what makes a life feel full, or grounded, or real. And then there’s the weird paradox of ambition, how we're taught to chase it like a finish line that keeps moving, constantly redefining itself right as you’re about to grasp it—because even when you get the job, or the degree, or the validation, there's always something next, something more, like a treadmill with invisible settings you didn’t realize were turned up all the way.++And have you noticed how we rarely stop to ask who set those definitions for us? Like was there ever a council that decided, yes, owning a home by thirty-five and retiring by sixty-five is the universal template for fulfillment? Or did it just accumulate like cultural sediment over generations, layered into us so deeply that questioning it feels uncomfortable, even dangerous? And isn’t it strange that we spend so much of our lives trying to optimize things—our workflows, our diets, our sleep, our morning routines—as though the point of life is to operate more efficiently rather than to experience it more richly? We build these intricate systems, these rulebooks for being a “high-functioning” human, but where in all of that is the space for feeling lost, for being soft, for wandering without a purpose just because it’s a sunny day and your heart is tugging you toward nowhere in particular?++Sometimes I lie awake at night and wonder if all the noise we wrap around ourselves—notifications, updates, performance reviews, even our internal monologues—might be crowding out the questions we were meant to live into slowly, like how to love better, or how to forgive ourselves, or what the hell we’re even doing here in the first place. And when you strip it all down—no goals, no KPIs, no curated identity—what’s actually left of us? Are we just a sum of the roles we perform, or is there something quieter underneath that we've forgotten how to hear?++And if there is something underneath all of it—something real, something worth listening to—then how do we begin to uncover it, gently, without rushing or reducing it to another task on our to-do list?+ """++ MAX_ATTEMPTS = 10+ prev_length = None++ for attempt in range(MAX_ATTEMPTS):+ client.agents.messages.create(+ agent_id=temp_agent_state.id,+ messages=[MessageCreate(role="user", content=philosophical_question)],+ )++ temp_agent_state = client.agents.retrieve(agent_id=temp_agent_state.id)+ message_ids = temp_agent_state.message_ids+ current_length = len(message_ids)++ print("LENGTH OF IN_CONTEXT_MESSAGES:", current_length)++ if prev_length is not None and current_length <= prev_length:+ # TODO: Add more stringent checks here+ print(f"Summarization was triggered, detected current_length {current_length} is at least prev_length {prev_length}.")+ break++ prev_length = current_length+ else:+ raise AssertionError("Summarization was not triggered after 10 messages")diff --git a/.github/scripts/model-sweep/supported-models.mdx b/.github/scripts/model-sweep/supported-models.mdxnew file mode 100644index 000000000..a8a203c70--- /dev/null+++ b/.github/scripts/model-sweep/supported-models.mdx@@ -0,0 +1,4551 @@+---+title: Support Models+generated: 2025-06-20T16:40:44.072054+---++# Supported Models++## Overview++Letta routinely runs automated scans against available providers and models. These are the results of the latest scan.++Ran 2464 tests against 154 models across 7 providers on June 20th, 2025+++## anthropic++| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|+| `claude-3-5-haiku-20241022` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_5_haiku_20241022_details) |+| `claude-3-5-sonnet-20241022` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_5_sonnet_20241022_details) |+| `claude-3-7-sonnet-20250219` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_7_sonnet_20250219_details) |+| `claude-sonnet-4-20250514` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_sonnet_4_20250514_details) |+| `claude-opus-4-20250514` | ✅ | ✅ | ⚠️ | 200,000 | 2025-06-20 | [View](#anthropic_claude_opus_4_20250514_details) |+| `claude-3-5-sonnet-20240620` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_5_sonnet_20240620_details) |+| `claude-3-haiku-20240307` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_haiku_20240307_details) |+| `claude-3-opus-20240229` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_opus_20240229_details) |+| `claude-3-sonnet-20240229` | ❌ | ❌ | ❌ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_sonnet_20240229_details) |+++## openai++| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|+| `gpt-4.1` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_details) |+| `gpt-4.1-2025-04-14` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_2025_04_14_details) |+| `gpt-4.1-nano-2025-04-14` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_nano_2025_04_14_details) |+| `gpt-4o` | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_details) |+| `gpt-4o-2024-05-13` | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_2024_05_13_details) |+| `gpt-4-turbo` | ✅ | ✅ | ⚠️ | 8,192 | 2025-06-20 | [View](#openai_gpt_4_turbo_details) |+| `gpt-4.1-mini` | ✅ | ✅ | ⚠️ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_mini_details) |+| `gpt-4.5-preview` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | [View](#openai_gpt_4.5_preview_details) |+| `gpt-4.5-preview-2025-02-27` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | [View](#openai_gpt_4.5_preview_2025_02_27_details) |+| `gpt-4o-2024-08-06` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_2024_08_06_details) |+| `gpt-4-0613` | ✅ | ✅ | ❌ | 8,192 | 2025-06-20 | [View](#openai_gpt_4_0613_details) |+| `gpt-4-1106-preview` | ✅ | ✅ | ❌ | 128,000 | 2025-06-20 | [View](#openai_gpt_4_1106_preview_details) |+| `gpt-4-turbo-2024-04-09` | ✅ | ⚠️ | ✅ | 128,000 | 2025-06-20 | [View](#openai_gpt_4_turbo_2024_04_09_details) |+| `gpt-4.1-mini-2025-04-14` | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_mini_2025_04_14_details) |+| `gpt-4.1-nano` | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_nano_details) |+| `gpt-4o-2024-11-20` | ⚠️ | ✅ | ✅ | 8,192 | 2025-06-20 | [View](#openai_gpt_4o_2024_11_20_details) |+| `gpt-4-turbo-preview` | ✅ | ⚠️ | ❌ | 128,000 | 2025-06-20 | [View](#openai_gpt_4_turbo_preview_details) |+| `gpt-4-0125-preview` | ⚠️ | ✅ | ❌ | 128,000 | 2025-06-20 | [View](#openai_gpt_4_0125_preview_details) |+| `gpt-4o-mini` | ⚠️ | ⚠️ | ⚠️ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_mini_details) |+| `gpt-4o-mini-2024-07-18` | ⚠️ | ⚠️ | ❌ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_mini_2024_07_18_details) |+| `gpt-4` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_gpt_4_details) |+| `o1` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o1_details) |+| `o1-2024-12-17` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o1_2024_12_17_details) |+| `o3` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_details) |+| `o3-2025-04-16` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_2025_04_16_details) |+| `o3-mini` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_mini_details) |+| `o3-mini-2025-01-31` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_mini_2025_01_31_details) |+| `o3-pro` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_pro_details) |+| `o3-pro-2025-06-10` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_pro_2025_06_10_details) |+++## google_ai++| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|+| `gemini-1.5-pro` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_pro_details) |+| `gemini-1.5-pro-002` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_pro_002_details) |+| `gemini-1.5-pro-latest` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_pro_latest_details) |+| `gemini-2.5-flash-preview-04-17-thinking` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_flash_preview_04_17_thinking_details) |+| `gemini-2.5-pro-preview-03-25` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_preview_03_25_details) |+| `gemini-2.5-pro-preview-05-06` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_preview_05_06_details) |+| `gemini-2.5-flash-preview-05-20` | ✅ | ⚠️ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_flash_preview_05_20_details) |+| `gemini-2.0-flash-thinking-exp` | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_thinking_exp_details) |+| `gemini-2.0-flash-thinking-exp-1219` | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_thinking_exp_1219_details) |+| `gemini-2.0-flash-thinking-exp-01-21` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_thinking_exp_01_21_details) |+| `gemini-2.5-flash-preview-04-17` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_flash_preview_04_17_details) |+| `gemini-2.5-pro-preview-06-05` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_preview_06_05_details) |+| `gemini-1.0-pro-vision-latest` | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | [View](#google_ai_gemini_1.0_pro_vision_latest_details) |+| `gemini-1.5-flash` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_details) |+| `gemini-1.5-flash-002` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_002_details) |+| `gemini-1.5-flash-8b` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_8b_details) |+| `gemini-1.5-flash-8b-001` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_8b_001_details) |+| `gemini-1.5-flash-8b-latest` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_8b_latest_details) |+| `gemini-1.5-flash-latest` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_latest_details) |+| `gemini-2.0-flash` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_details) |+| `gemini-2.0-flash-001` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_001_details) |+| `gemini-2.0-flash-exp` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_exp_details) |+| `gemini-2.0-flash-exp-image-generation` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_exp_image_generation_details) |+| `gemini-2.0-flash-lite` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_lite_details) |+| `gemini-2.0-flash-lite-001` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_lite_001_details) |+| `gemini-2.0-flash-lite-preview` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_lite_preview_details) |+| `gemini-2.0-flash-lite-preview-02-05` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_lite_preview_02_05_details) |+| `gemini-2.0-flash-preview-image-generation` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_preview_image_generation_details) |+| `gemini-2.0-pro-exp` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_pro_exp_details) |+| `gemini-2.0-pro-exp-02-05` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_pro_exp_02_05_details) |+| `gemini-2.5-flash-preview-tts` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#google_ai_gemini_2.5_flash_preview_tts_details) |+| `gemini-2.5-pro-exp-03-25` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_exp_03_25_details) |+| `gemini-2.5-pro-preview-tts` | ❌ | ❌ | ❌ | 65,536 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_preview_tts_details) |+| `gemini-exp-1206` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_exp_1206_details) |+| `gemini-pro-vision` | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | [View](#google_ai_gemini_pro_vision_details) |+++## letta++| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|+| `letta-free` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#letta_letta_free_details) |+++## together++| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|+| `Qwen/Qwen2.5-72B-Instruct-Turbo` | ✅ | ✅ | ⚠️ | 131,072 | 2025-06-20 | [View](#together_qwen_qwen2.5_72b_instruct_turbo_details) |+| `arcee-ai/virtuoso-large` | ⚠️ | ✅ | ✅ | 131,072 | 2025-06-20 | [View](#together_arcee_ai_virtuoso_large_details) |+| `Qwen/QwQ-32B` | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | [View](#together_qwen_qwq_32b_details) |+| `Qwen/Qwen2.5-7B-Instruct-Turbo` | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | [View](#together_qwen_qwen2.5_7b_instruct_turbo_details) |+| `Qwen/Qwen2.5-Coder-32B-Instruct` | ⚠️ | ✅ | ⚠️ | 16,384 | 2025-06-20 | [View](#together_qwen_qwen2.5_coder_32b_instruct_details) |+| `arcee-ai/coder-large` | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | [View](#together_arcee_ai_coder_large_details) |+| `arcee_ai/arcee-spotlight` | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | [View](#together_arcee_ai_arcee_spotlight_details) |+| `meta-llama/Llama-3.2-3B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.2_3b_instruct_turbo_details) |+| `meta-llama/Llama-3.3-70B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.3_70b_instruct_turbo_details) |+| `meta-llama/Llama-3.3-70B-Instruct-Turbo-Free` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.3_70b_instruct_turbo_free_details) |+| `meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 130,815 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details) |+| `meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details) |+| `nvidia/Llama-3.1-Nemotron-70B-Instruct-HF` | ⚠️ | ✅ | ❌ | 32,768 | 2025-06-20 | [View](#together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details) |+| `arcee-ai/virtuoso-medium-v2` | ⚠️ | ⚠️ | ✅ | 131,072 | 2025-06-20 | [View](#together_arcee_ai_virtuoso_medium_v2_details) |+| `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8` | ⚠️ | ❌ | ✅ | 1,048,576 | 2025-06-20 | [View](#together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details) |+| `Qwen/Qwen3-235B-A22B-fp8-tput` | ⚠️ | ⚠️ | ❌ | 40,960 | 2025-06-20 | [View](#together_qwen_qwen3_235b_a22b_fp8_tput_details) |+| `deepseek-ai/DeepSeek-V3` | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_v3_details) |+| `meta-llama/Llama-4-Scout-17B-16E-Instruct` | ⚠️ | ⚠️ | ❌ | 1,048,576 | 2025-06-20 | [View](#together_meta_llama_llama_4_scout_17b_16e_instruct_details) |+| `meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo` | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details) |+| `mistralai/Mixtral-8x7B-Instruct-v0.1` | ⚠️ | ⚠️ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mixtral_8x7b_instruct_v0.1_details) |+| `arcee-ai/caller` | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | [View](#together_arcee_ai_caller_details) |+| `mistralai/Mistral-Small-24B-Instruct-2501` | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mistral_small_24b_instruct_2501_details) |+| `NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details) |+| `Qwen/Qwen2-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_qwen_qwen2_72b_instruct_details) |+| `Qwen/Qwen2-VL-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_qwen_qwen2_vl_72b_instruct_details) |+| `Qwen/Qwen2.5-VL-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_qwen_qwen2.5_vl_72b_instruct_details) |+| `arcee-ai/arcee-blitz` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_arcee_ai_arcee_blitz_details) |+| `arcee-ai/maestro-reasoning` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_arcee_ai_maestro_reasoning_details) |+| `deepseek-ai/DeepSeek-R1` | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_details) |+| `deepseek-ai/DeepSeek-R1-Distill-Llama-70B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_distill_llama_70b_details) |+| `deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details) |+| `deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details) |+| `deepseek-ai/DeepSeek-R1-Distill-Qwen-14B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_distill_qwen_14b_details) |+| `deepseek-ai/DeepSeek-V3-p-dp` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_v3_p_dp_details) |+| `google/gemma-2-27b-it` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_google_gemma_2_27b_it_details) |+| `lgai/exaone-3-5-32b-instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_lgai_exaone_3_5_32b_instruct_details) |+| `lgai/exaone-deep-32b` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_lgai_exaone_deep_32b_details) |+| `marin-community/marin-8b-instruct` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_marin_community_marin_8b_instruct_details) |+| `meta-llama/Llama-3-70b-chat-hf` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_meta_llama_llama_3_70b_chat_hf_details) |+| `meta-llama/Llama-3-8b-chat-hf` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_meta_llama_llama_3_8b_chat_hf_details) |+| `meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details) |+| `meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details) |+| `meta-llama/Llama-Vision-Free` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_vision_free_details) |+| `meta-llama/Meta-Llama-3-70B-Instruct-Turbo` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3_70b_instruct_turbo_details) |+| `meta-llama/Meta-Llama-3-8B-Instruct-Lite` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3_8b_instruct_lite_details) |+| `mistralai/Mistral-7B-Instruct-v0.1` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mistral_7b_instruct_v0.1_details) |+| `mistralai/Mistral-7B-Instruct-v0.2` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mistral_7b_instruct_v0.2_details) |+| `mistralai/Mistral-7B-Instruct-v0.3` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mistral_7b_instruct_v0.3_details) |+| `perplexity-ai/r1-1776` | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | [View](#together_perplexity_ai_r1_1776_details) |+| `scb10x/scb10x-llama3-1-typhoon2-70b-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details) |+| `scb10x/scb10x-typhoon-2-1-gemma3-12b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details) |+| `togethercomputer/MoA-1` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_togethercomputer_moa_1_details) |+| `togethercomputer/MoA-1-Turbo` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_togethercomputer_moa_1_turbo_details) |+| `togethercomputer/Refuel-Llm-V2` | ❌ | ❌ | ❌ | 16,384 | 2025-06-20 | [View](#together_togethercomputer_refuel_llm_v2_details) |+| `togethercomputer/Refuel-Llm-V2-Small` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_togethercomputer_refuel_llm_v2_small_details) |+++## deepseek++| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|+| `deepseek-chat` | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | [View](#deepseek_deepseek_chat_details) |+| `deepseek-reasoner` | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | [View](#deepseek_deepseek_reasoner_details) |+++## groq++| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|+| `allam-2-7b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_allam_2_7b_details) |+| `compound-beta` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_compound_beta_details) |+| `compound-beta-mini` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_compound_beta_mini_details) |+| `deepseek-r1-distill-llama-70b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_deepseek_r1_distill_llama_70b_details) |+| `distil-whisper-large-v3-en` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_distil_whisper_large_v3_en_details) |+| `gemma2-9b-it` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_gemma2_9b_it_details) |+| `llama-3.1-8b-instant` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama_3.1_8b_instant_details) |+| `llama-3.3-70b-versatile` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama_3.3_70b_versatile_details) |+| `llama-guard-3-8b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama_guard_3_8b_details) |+| `llama3-70b-8192` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama3_70b_8192_details) |+| `llama3-8b-8192` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama3_8b_8192_details) |+| `meta-llama/llama-4-maverick-17b-128e-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_4_maverick_17b_128e_instruct_details) |+| `meta-llama/llama-4-scout-17b-16e-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_4_scout_17b_16e_instruct_details) |+| `meta-llama/llama-guard-4-12b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_guard_4_12b_details) |+| `meta-llama/llama-prompt-guard-2-22m` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_prompt_guard_2_22m_details) |+| `meta-llama/llama-prompt-guard-2-86m` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_prompt_guard_2_86m_details) |+| `mistral-saba-24b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_mistral_saba_24b_details) |+| `playai-tts` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_playai_tts_details) |+| `playai-tts-arabic` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_playai_tts_arabic_details) |+| `qwen-qwq-32b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_qwen_qwq_32b_details) |+| `qwen/qwen3-32b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_qwen_qwen3_32b_details) |+| `whisper-large-v3` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_whisper_large_v3_details) |+| `whisper-large-v3-turbo` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_whisper_large_v3_turbo_details) |+++---++# Detailed Test Results++## anthropic++<a id="anthropic_claude_3_5_haiku_20241022_details"></a>+### claude-3-5-haiku-20241022++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="anthropic_claude_3_5_sonnet_20241022_details"></a>+### claude-3-5-sonnet-20241022++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="anthropic_claude_3_7_sonnet_20250219_details"></a>+### claude-3-7-sonnet-20250219++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="anthropic_claude_sonnet_4_20250514_details"></a>+### claude-sonnet-4-20250514++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="anthropic_claude_opus_4_20250514_details"></a>+### claude-opus-4-20250514++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="anthropic_claude_3_5_sonnet_20240620_details"></a>+### claude-3-5-sonnet-20240620++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="anthropic_claude_3_haiku_20240307_details"></a>+### claude-3-haiku-20240307++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="anthropic_claude_3_opus_20240229_details"></a>+### claude-3-opus-20240229++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="anthropic_claude_3_sonnet_20240229_details"></a>+### claude-3-sonnet-20240229++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++++## openai++<a id="openai_gpt_4.1_details"></a>+### gpt-4.1++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4.1_2025_04_14_details"></a>+### gpt-4.1-2025-04-14++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4.1_nano_2025_04_14_details"></a>+### gpt-4.1-nano-2025-04-14++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4o_details"></a>+### gpt-4o++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4o_2024_05_13_details"></a>+### gpt-4o-2024-05-13++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4_turbo_details"></a>+### gpt-4-turbo++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4.1_mini_details"></a>+### gpt-4.1-mini++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4.5_preview_details"></a>+### gpt-4.5-preview++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4.5_preview_2025_02_27_details"></a>+### gpt-4.5-preview-2025-02-27++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4o_2024_08_06_details"></a>+### gpt-4o-2024-08-06++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4_0613_details"></a>+### gpt-4-0613++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4_1106_preview_details"></a>+### gpt-4-1106-preview++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4_turbo_2024_04_09_details"></a>+### gpt-4-turbo-2024-04-09++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4.1_mini_2025_04_14_details"></a>+### gpt-4.1-mini-2025-04-14++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4.1_nano_details"></a>+### gpt-4.1-nano++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4o_2024_11_20_details"></a>+### gpt-4o-2024-11-20++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4_turbo_preview_details"></a>+### gpt-4-turbo-preview++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4_0125_preview_details"></a>+### gpt-4-0125-preview++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4o_mini_details"></a>+### gpt-4o-mini++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4o_mini_2024_07_18_details"></a>+### gpt-4o-mini-2024-07-18++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_gpt_4_details"></a>+### gpt-4++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_o1_details"></a>+### o1++### Basic++- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_o1_2024_12_17_details"></a>+### o1-2024-12-17++### Basic++- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_o3_details"></a>+### o3++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_o3_2025_04_16_details"></a>+### o3-2025-04-16++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_o3_mini_details"></a>+### o3-mini++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_o3_mini_2025_01_31_details"></a>+### o3-mini-2025-01-31++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_o3_pro_details"></a>+### o3-pro++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="openai_o3_pro_2025_06_10_details"></a>+### o3-pro-2025-06-10++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++++## google_ai++<a id="google_ai_gemini_1.5_pro_details"></a>+### gemini-1.5-pro++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_1.5_pro_002_details"></a>+### gemini-1.5-pro-002++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_1.5_pro_latest_details"></a>+### gemini-1.5-pro-latest++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.5_flash_preview_04_17_thinking_details"></a>+### gemini-2.5-flash-preview-04-17-thinking++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.5_pro_preview_03_25_details"></a>+### gemini-2.5-pro-preview-03-25++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.5_pro_preview_05_06_details"></a>+### gemini-2.5-pro-preview-05-06++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.5_flash_preview_05_20_details"></a>+### gemini-2.5-flash-preview-05-20++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.0_flash_thinking_exp_details"></a>+### gemini-2.0-flash-thinking-exp++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.0_flash_thinking_exp_1219_details"></a>+### gemini-2.0-flash-thinking-exp-1219++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.0_flash_thinking_exp_01_21_details"></a>+### gemini-2.0-flash-thinking-exp-01-21++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.5_flash_preview_04_17_details"></a>+### gemini-2.5-flash-preview-04-17++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.5_pro_preview_06_05_details"></a>+### gemini-2.5-pro-preview-06-05++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_1.0_pro_vision_latest_details"></a>+### gemini-1.0-pro-vision-latest++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_1.5_flash_details"></a>+### gemini-1.5-flash++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_1.5_flash_002_details"></a>+### gemini-1.5-flash-002++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_1.5_flash_8b_details"></a>+### gemini-1.5-flash-8b++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_1.5_flash_8b_001_details"></a>+### gemini-1.5-flash-8b-001++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_1.5_flash_8b_latest_details"></a>+### gemini-1.5-flash-8b-latest++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_1.5_flash_latest_details"></a>+### gemini-1.5-flash-latest++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.0_flash_details"></a>+### gemini-2.0-flash++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.0_flash_001_details"></a>+### gemini-2.0-flash-001++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.0_flash_exp_details"></a>+### gemini-2.0-flash-exp++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.0_flash_exp_image_generation_details"></a>+### gemini-2.0-flash-exp-image-generation++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.0_flash_lite_details"></a>+### gemini-2.0-flash-lite++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.0_flash_lite_001_details"></a>+### gemini-2.0-flash-lite-001++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.0_flash_lite_preview_details"></a>+### gemini-2.0-flash-lite-preview++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.0_flash_lite_preview_02_05_details"></a>+### gemini-2.0-flash-lite-preview-02-05++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.0_flash_preview_image_generation_details"></a>+### gemini-2.0-flash-preview-image-generation++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.0_pro_exp_details"></a>+### gemini-2.0-pro-exp++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.0_pro_exp_02_05_details"></a>+### gemini-2.0-pro-exp-02-05++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.5_flash_preview_tts_details"></a>+### gemini-2.5-flash-preview-tts++### Basic++- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.5_pro_exp_03_25_details"></a>+### gemini-2.5-pro-exp-03-25++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_2.5_pro_preview_tts_details"></a>+### gemini-2.5-pro-preview-tts++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_exp_1206_details"></a>+### gemini-exp-1206++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="google_ai_gemini_pro_vision_details"></a>+### gemini-pro-vision++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++++## letta++<a id="letta_letta_free_details"></a>+### letta-free++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++++## together++<a id="together_qwen_qwen2.5_72b_instruct_turbo_details"></a>+### Qwen/Qwen2.5-72B-Instruct-Turbo++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_arcee_ai_virtuoso_large_details"></a>+### arcee-ai/virtuoso-large++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_qwen_qwq_32b_details"></a>+### Qwen/QwQ-32B++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_qwen_qwen2.5_7b_instruct_turbo_details"></a>+### Qwen/Qwen2.5-7B-Instruct-Turbo++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_qwen_qwen2.5_coder_32b_instruct_details"></a>+### Qwen/Qwen2.5-Coder-32B-Instruct++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_arcee_ai_coder_large_details"></a>+### arcee-ai/coder-large++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_arcee_ai_arcee_spotlight_details"></a>+### arcee_ai/arcee-spotlight++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_meta_llama_llama_3.2_3b_instruct_turbo_details"></a>+### meta-llama/Llama-3.2-3B-Instruct-Turbo++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_meta_llama_llama_3.3_70b_instruct_turbo_details"></a>+### meta-llama/Llama-3.3-70B-Instruct-Turbo++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_meta_llama_llama_3.3_70b_instruct_turbo_free_details"></a>+### meta-llama/Llama-3.3-70B-Instruct-Turbo-Free++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details"></a>+### meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details"></a>+### meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details"></a>+### nvidia/Llama-3.1-Nemotron-70B-Instruct-HF++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_arcee_ai_virtuoso_medium_v2_details"></a>+### arcee-ai/virtuoso-medium-v2++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details"></a>+### meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_qwen_qwen3_235b_a22b_fp8_tput_details"></a>+### Qwen/Qwen3-235B-A22B-fp8-tput++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_deepseek_ai_deepseek_v3_details"></a>+### deepseek-ai/DeepSeek-V3++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_meta_llama_llama_4_scout_17b_16e_instruct_details"></a>+### meta-llama/Llama-4-Scout-17B-16E-Instruct++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details"></a>+### meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_mistralai_mixtral_8x7b_instruct_v0.1_details"></a>+### mistralai/Mixtral-8x7B-Instruct-v0.1++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_arcee_ai_caller_details"></a>+### arcee-ai/caller++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_mistralai_mistral_small_24b_instruct_2501_details"></a>+### mistralai/Mistral-Small-24B-Instruct-2501++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details"></a>+### NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_qwen_qwen2_72b_instruct_details"></a>+### Qwen/Qwen2-72B-Instruct++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_qwen_qwen2_vl_72b_instruct_details"></a>+### Qwen/Qwen2-VL-72B-Instruct++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_qwen_qwen2.5_vl_72b_instruct_details"></a>+### Qwen/Qwen2.5-VL-72B-Instruct++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_arcee_ai_arcee_blitz_details"></a>+### arcee-ai/arcee-blitz++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_arcee_ai_maestro_reasoning_details"></a>+### arcee-ai/maestro-reasoning++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_deepseek_ai_deepseek_r1_details"></a>+### deepseek-ai/DeepSeek-R1++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_deepseek_ai_deepseek_r1_distill_llama_70b_details"></a>+### deepseek-ai/DeepSeek-R1-Distill-Llama-70B++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details"></a>+### deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details"></a>+### deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_deepseek_ai_deepseek_r1_distill_qwen_14b_details"></a>+### deepseek-ai/DeepSeek-R1-Distill-Qwen-14B++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_deepseek_ai_deepseek_v3_p_dp_details"></a>+### deepseek-ai/DeepSeek-V3-p-dp++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_google_gemma_2_27b_it_details"></a>+### google/gemma-2-27b-it++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_lgai_exaone_3_5_32b_instruct_details"></a>+### lgai/exaone-3-5-32b-instruct++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_lgai_exaone_deep_32b_details"></a>+### lgai/exaone-deep-32b++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_marin_community_marin_8b_instruct_details"></a>+### marin-community/marin-8b-instruct++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_meta_llama_llama_3_70b_chat_hf_details"></a>+### meta-llama/Llama-3-70b-chat-hf++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_meta_llama_llama_3_8b_chat_hf_details"></a>+### meta-llama/Llama-3-8b-chat-hf++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details"></a>+### meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details"></a>+### meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_meta_llama_llama_vision_free_details"></a>+### meta-llama/Llama-Vision-Free++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_meta_llama_meta_llama_3_70b_instruct_turbo_details"></a>+### meta-llama/Meta-Llama-3-70B-Instruct-Turbo++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_meta_llama_meta_llama_3_8b_instruct_lite_details"></a>+### meta-llama/Meta-Llama-3-8B-Instruct-Lite++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_mistralai_mistral_7b_instruct_v0.1_details"></a>+### mistralai/Mistral-7B-Instruct-v0.1++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_mistralai_mistral_7b_instruct_v0.2_details"></a>+### mistralai/Mistral-7B-Instruct-v0.2++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_mistralai_mistral_7b_instruct_v0.3_details"></a>+### mistralai/Mistral-7B-Instruct-v0.3++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_perplexity_ai_r1_1776_details"></a>+### perplexity-ai/r1-1776++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details"></a>+### scb10x/scb10x-llama3-1-typhoon2-70b-instruct++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details"></a>+### scb10x/scb10x-typhoon-2-1-gemma3-12b++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_togethercomputer_moa_1_details"></a>+### togethercomputer/MoA-1++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_togethercomputer_moa_1_turbo_details"></a>+### togethercomputer/MoA-1-Turbo++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_togethercomputer_refuel_llm_v2_details"></a>+### togethercomputer/Refuel-Llm-V2++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="together_togethercomputer_refuel_llm_v2_small_details"></a>+### togethercomputer/Refuel-Llm-V2-Small++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++++## deepseek++<a id="deepseek_deepseek_chat_details"></a>+### deepseek-chat++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="deepseek_deepseek_reasoner_details"></a>+### deepseek-reasoner++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++++## groq++<a id="groq_allam_2_7b_details"></a>+### allam-2-7b++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_compound_beta_details"></a>+### compound-beta++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_compound_beta_mini_details"></a>+### compound-beta-mini++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_deepseek_r1_distill_llama_70b_details"></a>+### deepseek-r1-distill-llama-70b++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_distil_whisper_large_v3_en_details"></a>+### distil-whisper-large-v3-en++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_gemma2_9b_it_details"></a>+### gemma2-9b-it++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_llama_3.1_8b_instant_details"></a>+### llama-3.1-8b-instant++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_llama_3.3_70b_versatile_details"></a>+### llama-3.3-70b-versatile++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_llama_guard_3_8b_details"></a>+### llama-guard-3-8b++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_llama3_70b_8192_details"></a>+### llama3-70b-8192++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_llama3_8b_8192_details"></a>+### llama3-8b-8192++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_meta_llama_llama_4_maverick_17b_128e_instruct_details"></a>+### meta-llama/llama-4-maverick-17b-128e-instruct++### Basic++- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_meta_llama_llama_4_scout_17b_16e_instruct_details"></a>+### meta-llama/llama-4-scout-17b-16e-instruct++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_meta_llama_llama_guard_4_12b_details"></a>+### meta-llama/llama-guard-4-12b++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_meta_llama_llama_prompt_guard_2_22m_details"></a>+### meta-llama/llama-prompt-guard-2-22m++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_meta_llama_llama_prompt_guard_2_86m_details"></a>+### meta-llama/llama-prompt-guard-2-86m++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_mistral_saba_24b_details"></a>+### mistral-saba-24b++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_playai_tts_details"></a>+### playai-tts++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_playai_tts_arabic_details"></a>+### playai-tts-arabic++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_qwen_qwq_32b_details"></a>+### qwen-qwq-32b++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_qwen_qwen3_32b_details"></a>+### qwen/qwen3-32b++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_whisper_large_v3_details"></a>+### whisper-large-v3++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)++<a id="groq_whisper_large_v3_turbo_details"></a>+### whisper-large-v3-turbo++### Basic++- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)++### Token Streaming++- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)++### Multimodal++- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)diff --git a/.github/workflows/model-sweep.yaml b/.github/workflows/model-sweep.yamlnew file mode 100644index 000000000..e31d14268--- /dev/null+++ b/.github/workflows/model-sweep.yaml@@ -0,0 +1,144 @@+name: Model Sweep+on:+ workflow_dispatch:+ inputs:+ branch-name:+ required: true+ type: string++jobs:+ model-sweep:+ runs-on: [self-hosted, medium]+ services:+ qdrant:+ image: qdrant/qdrant+ ports:+ - 6333:6333+ postgres:+ image: pgvector/pgvector:pg17+ ports:+ - 5432:5432+ env:+ POSTGRES_HOST_AUTH_METHOD: trust+ POSTGRES_DB: postgres+ POSTGRES_USER: postgres+ options: >-+ --health-cmd pg_isready+ --health-interval 10s+ --health-timeout 5s+ --health-retries 5++ steps:+ - name: Check if gh is installed+ run: |+ if ! command -v gh >/dev/null 2>&1+ then+ echo "gh could not be found, installing now..."+ # install gh cli+ (type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) \+ && sudo mkdir -p -m 755 /etc/apt/keyrings \+ && out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \+ && cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \+ && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \+ && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \+ && sudo apt update \+ && sudo apt install gh -y+ fi++ - name: Checkout+ uses: actions/checkout@v4++ - name: Inject env vars into environment+ run: |+ # Get secrets and mask them before adding to environment+ while IFS= read -r line || [[ -n "$line" ]]; do+ if [[ -n "$line" ]]; then+ value=$(echo "$line" | cut -d= -f2-)+ echo "::add-mask::$value"+ echo "$line" >> $GITHUB_ENV+ fi+ done < <(letta_secrets_helper --env dev --service ci)++ - name: Install dependencies+ shell: bash+ run: poetry install --no-interaction --no-root ${{ inputs.install-args || '-E dev -E postgres -E external-tools -E tests -E cloud-tool-sandbox -E google' }}+ - name: Migrate database+ env:+ LETTA_PG_PORT: 5432+ LETTA_PG_USER: postgres+ LETTA_PG_PASSWORD: postgres+ LETTA_PG_DB: postgres+ LETTA_PG_HOST: localhost+ run: |+ psql -h localhost -U postgres -d postgres -c 'CREATE EXTENSION vector'+ poetry run alembic upgrade head++ - name: Run integration tests+ # if any of the 1000+ test cases fail, pytest reports exit code 1 and won't procces/upload the results+ continue-on-error: true+ env:+ LETTA_PG_PORT: 5432+ LETTA_PG_USER: postgres+ LETTA_PG_PASSWORD: postgres+ LETTA_PG_DB: postgres+ LETTA_PG_HOST: localhost+ LETTA_SERVER_PASS: test_server_token+ OPENAI_API_KEY: ${{ env.OPENAI_API_KEY }}+ ANTHROPIC_API_KEY: ${{ env.ANTHROPIC_API_KEY }}+ AZURE_API_KEY: ${{ env.AZURE_API_KEY }}+ AZURE_BASE_URL: ${{ secrets.AZURE_BASE_URL }}+ GEMINI_API_KEY: ${{ env.GEMINI_API_KEY }}+ COMPOSIO_API_KEY: ${{ env.COMPOSIO_API_KEY }}+ GOOGLE_CLOUD_PROJECT: ${{ secrets.GOOGLE_CLOUD_PROJECT}}+ GOOGLE_CLOUD_LOCATION: ${{ secrets.GOOGLE_CLOUD_LOCATION}}+ DEEPSEEK_API_KEY: ${{ env.DEEPSEEK_API_KEY}}+ LETTA_USE_EXPERIMENTAL: 1+ run: |+ poetry run pytest \+ -s -vv \+ .github/scripts/model-sweep/model_sweep.py \+ --json-report --json-report-file=.github/scripts/model-sweep/model_sweep_report.json --json-report-indent=4++ - name: Convert report to markdown+ continue-on-error: true+ # file path args to generate_model_sweep_markdown.py are relative to the script+ run: |+ poetry run python \+ .github/scripts/model-sweep/generate_model_sweep_markdown.py \+ .github/scripts/model-sweep/model_sweep_report.json \+ .github/scripts/model-sweep/supported-models.mdx+ echo "Model sweep report saved to .github/scripts/model-sweep/supported-models.mdx"++ - id: date+ run: echo "date=$(date +%Y-%m-%d)" >> $GITHUB_OUTPUT++ - name: commit and open pull request+ env:+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}+ run: |+ BRANCH_NAME=model-sweep/${{ inputs.branch-name || format('{0}', steps.date.outputs.date) }}+ gh auth setup-git+ git config --global user.name "github-actions[bot]"+ git config --global user.email "github-actions[bot]@users.noreply.github.com"+ git checkout -b $BRANCH_NAME+ git add .github/scripts/model-sweep/supported-models.mdx+ git commit -m "Update model sweep report"+ # only push if changes were made+ if git diff main --quiet; then+ echo "No changes detected, skipping push"+ exit 0+ else+ git push origin $BRANCH_NAME+ gh pr create \+ --base main \+ --head $BRANCH_NAME \+ --title "chore: update model sweep report" \+ --body "Automated PR to update model sweep report"+ fi++ - name: Upload model sweep report+ if: always()+ uses: actions/upload-artifact@v4+ with:+ name: model-sweep-report+ path: .github/scripts/model-sweep/model_sweep_report.jsondiff --git a/.github/workflows/send-message-integration-tests.yaml b/.github/workflows/send-message-integration-tests.yamlnew file mode 100644index 000000000..eee96265d--- /dev/null+++ b/.github/workflows/send-message-integration-tests.yaml@@ -0,0 +1,155 @@+name: Send Message SDK Tests+on:+ pull_request_target:+ # branches: [main] # TODO: uncomment before merge+ types: [labeled]+ paths:+ - 'letta/**'++jobs:+ send-messages:+ # Only run when the "safe to test" label is applied+ if: contains(github.event.pull_request.labels.*.name, 'safe to test')+ runs-on: ubuntu-latest+ timeout-minutes: 15+ strategy:+ fail-fast: false+ matrix:+ config_file:+ - "openai-gpt-4o-mini.json"+ - "azure-gpt-4o-mini.json"+ - "claude-3-5-sonnet.json"+ - "claude-3-7-sonnet.json"+ - "claude-3-7-sonnet-extended.json"+ - "gemini-pro.json"+ - "gemini-vertex.json"+ services:+ qdrant:+ image: qdrant/qdrant+ ports:+ - 6333:6333+ postgres:+ image: pgvector/pgvector:pg17+ ports:+ - 5432:5432+ env:+ POSTGRES_HOST_AUTH_METHOD: trust+ POSTGRES_DB: postgres+ POSTGRES_USER: postgres+ options: >-+ --health-cmd pg_isready+ --health-interval 10s+ --health-timeout 5s+ --health-retries 5++ steps:+ # Ensure secrets don't leak+ - name: Configure git to hide secrets+ run: |+ git config --global core.logAllRefUpdates false+ git config --global log.hideCredentials true+ - name: Set up secret masking+ run: |+ # Automatically mask any environment variable ending with _KEY+ for var in $(env | grep '_KEY=' | cut -d= -f1); do+ value="${!var}"+ if [[ -n "$value" ]]; then+ # Mask the full value+ echo "::add-mask::$value"++ # Also mask partial values (first and last several characters)+ # This helps when only parts of keys appear in logs+ if [[ ${#value} -gt 8 ]]; then+ echo "::add-mask::${value:0:8}"+ echo "::add-mask::${value:(-8)}"+ fi++ # Also mask with common formatting changes+ # Some logs might add quotes or other characters+ echo "::add-mask::\"$value\""+ echo "::add-mask::$value\""+ echo "::add-mask::\"$value"++ echo "Masked secret: $var (length: ${#value})"+ fi+ done++ # Check out base repository code, not the PR's code (for security)+ - name: Checkout base repository+ uses: actions/checkout@v4 # No ref specified means it uses base branch++ # Only extract relevant files from the PR (for security, specifically prevent modification of workflow files)+ - name: Extract PR schema files+ run: |+ # Fetch PR without checking it out+ git fetch origin pull/${{ github.event.pull_request.number }}/head:pr-${{ github.event.pull_request.number }}++ # Extract ONLY the schema files+ git checkout pr-${{ github.event.pull_request.number }} -- letta/+ - name: Set up python 3.12+ id: setup-python+ uses: actions/setup-python@v5+ with:+ python-version: 3.12+ - name: Load cached Poetry Binary+ id: cached-poetry-binary+ uses: actions/cache@v4+ with:+ path: ~/.local+ key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-1.8.3+ - name: Install Poetry+ uses: snok/install-poetry@v1+ with:+ version: 1.8.3+ virtualenvs-create: true+ virtualenvs-in-project: true+ - name: Load cached venv+ id: cached-poetry-dependencies+ uses: actions/cache@v4+ with:+ path: .venv+ key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}${{ inputs.install-args || '-E dev -E postgres -E external-tools -E tests -E cloud-tool-sandbox' }}+ # Restore cache with this prefix if not exact match with key+ # Note cache-hit returns false in this case, so the below step will run+ restore-keys: |+ venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-+ - name: Install dependencies+ if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'+ shell: bash+ run: poetry install --no-interaction --no-root ${{ inputs.install-args || '-E dev -E postgres -E external-tools -E tests -E cloud-tool-sandbox -E google' }}+ - name: Install letta packages via Poetry+ run: |+ poetry run pip install --upgrade letta-client letta+ - name: Migrate database+ env:+ LETTA_PG_PORT: 5432+ LETTA_PG_USER: postgres+ LETTA_PG_PASSWORD: postgres+ LETTA_PG_DB: postgres+ LETTA_PG_HOST: localhost+ run: |+ psql -h localhost -U postgres -d postgres -c 'CREATE EXTENSION vector'+ poetry run alembic upgrade head+ - name: Run integration tests for ${{ matrix.config_file }}+ env:+ LLM_CONFIG_FILE: ${{ matrix.config_file }}+ LETTA_PG_PORT: 5432+ LETTA_PG_USER: postgres+ LETTA_PG_PASSWORD: postgres+ LETTA_PG_DB: postgres+ LETTA_PG_HOST: localhost+ LETTA_SERVER_PASS: test_server_token+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}+ AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}+ AZURE_BASE_URL: ${{ secrets.AZURE_BASE_URL }}+ GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}+ COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }}+ DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}+ GOOGLE_CLOUD_PROJECT: ${{ secrets.GOOGLE_CLOUD_PROJECT }}+ GOOGLE_CLOUD_LOCATION: ${{ secrets.GOOGLE_CLOUD_LOCATION }}+ run: |+ poetry run pytest \+ -s -vv \+ tests/integration_test_send_message.py \+ --maxfail=1 --durations=10diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 760be895f..3ea08c3c6 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -28,7 +28,7 @@ First, install Poetry using [the official instructions here](https://python-poetOnce Poetry is installed, navigate to the letta directory and install the Letta project with Poetry:```shellcd letta-poetry shell+eval $(poetry env activate)poetry install --all-extras```#### Setup PostgreSQL environment (optional)diff --git a/README.md b/README.mdindex 042d476e8..1698d18c7 100644--- a/README.md+++ b/README.md@@ -8,26 +8,13 @@<div align="center"><h1>Letta (previously MemGPT)</h1>--**☄️ New release: Letta Agent Development Environment (_read more [here](#-access-the-ade-agent-development-environment)_) ☄️**--<p align="center">- <picture>- <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/letta-ai/letta/refs/heads/main/assets/example_ade_screenshot.png">- <source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/letta-ai/letta/refs/heads/main/assets/example_ade_screenshot_light.png">- <img alt="Letta logo" src="https://raw.githubusercontent.com/letta-ai/letta/refs/heads/main/assets/example_ade_screenshot.png" width="800">- </picture>-</p>------<h3>[Homepage](https://letta.com) // [Documentation](https://docs.letta.com) // [ADE](https://docs.letta.com/agent-development-environment) // [Letta Cloud](https://forms.letta.com/early-access)</h3>-**👾 Letta** is an open source framework for building stateful LLM applications. You can use Letta to build **stateful agents** with advanced reasoning capabilities and transparent long-term memory. The Letta framework is white box and model-agnostic.+**👾 Letta** is an open source framework for building **stateful agents** with advanced reasoning capabilities and transparent long-term memory. The Letta framework is white box and model-agnostic.[](https://discord.gg/letta)[](https://twitter.com/Letta_AI)@@ -157,7 +144,7 @@ No, the data in your Letta server database stays on your machine. The Letta ADE> _"Do I have to use your ADE? Can I build my own?"_-The ADE is built on top of the (fully open source) Letta server and Letta Agents API. You can build your own application like the ADE on top of the REST API (view the documention [here](https://docs.letta.com/api-reference)).+The ADE is built on top of the (fully open source) Letta server and Letta Agents API. You can build your own application like the ADE on top of the REST API (view the documentation [here](https://docs.letta.com/api-reference)).> _"Can I interact with Letta agents via the CLI?"_diff --git a/dev-compose.yaml b/dev-compose.yamlindex 42239b987..36fd5c549 100644--- a/dev-compose.yaml+++ b/dev-compose.yaml@@ -28,7 +28,6 @@ services:- "8083:8083"- "8283:8283"environment:- - SERPAPI_API_KEY=${SERPAPI_API_KEY}- LETTA_PG_DB=${LETTA_PG_DB:-letta}- LETTA_PG_USER=${LETTA_PG_USER:-letta}- LETTA_PG_PASSWORD=${LETTA_PG_PASSWORD:-letta}diff --git a/examples/docs/example.py b/examples/docs/example.pyindex 2f3530a28..990d846d6 100644--- a/examples/docs/example.py+++ b/examples/docs/example.py@@ -8,6 +8,8 @@ If you're using Letta Cloud, replace 'baseURL' with 'token'See: https://docs.letta.com/api-reference/overviewExecute this script using `poetry run python3 example.py`++This will install `letta_client` and other dependencies."""client = Letta(base_url="http://localhost:8283",diff --git a/examples/files/README.md b/examples/files/README.mdnew file mode 100644index 000000000..e6b4a421c--- /dev/null+++ b/examples/files/README.md@@ -0,0 +1,34 @@+# Letta Files and Streaming Demo++This demo shows how to work with Letta's file upload and streaming capabilities.++## Features++- Upload files from disk to a Letta data source+- Create files from strings and upload them+- Download and upload PDF files+- Create an agent and attach data sources+- Stream agent responses in real-time+- Interactive chat with file-aware agent++## Files++- `main.py` - Main demo script showing file upload and streaming+- `example-on-disk.txt` - Sample text file for upload demonstration+- `memgpt.pdf` - MemGPT paper (downloaded automatically)++## Setup++1. Set your Letta API key: `export LETTA_API_KEY=your_key_here`+2. Install dependencies: `pip install letta-client requests rich`+3. Run the demo: `python main.py`++## Usage++The demo will:+1. Create a data source called "Example Source"+2. Upload the example text file and PDF+3. Create an agent named "Clippy"+4. Start an interactive chat session++Type 'quit' or 'exit' to end the conversation.diff --git a/examples/files/example-on-disk.txt b/examples/files/example-on-disk.txtnew file mode 100644index 000000000..d8f9b2b7a--- /dev/null+++ b/examples/files/example-on-disk.txt@@ -0,0 +1,2 @@+Hey, you're looking at a different example.+This password is "stateful agents".diff --git a/examples/files/main.py b/examples/files/main.pynew file mode 100644index 000000000..6ef049784--- /dev/null+++ b/examples/files/main.py@@ -0,0 +1,190 @@+"""+Letta Filesystem++This demo shows how to:+1. Create a folder and upload files (both from disk and from strings)+2. Create an agent and attach the data folder+3. Stream the agent's responses+4. Query the agent about the uploaded files++The demo uploads:+- A text file from disk (example-on-disk.txt)+- A text file created from a string (containing a password)+- The MemGPT paper PDF from arXiv++Then asks the agent to summarize the paper and find passwords in the files.+"""++import os++import requests+from letta_client import Letta+from letta_client.core.api_error import ApiError+from rich import print++LETTA_API_KEY = os.getenv("LETTA_API_KEY")+if LETTA_API_KEY is None:+ raise ValueError("LETTA_API_KEY is not set")++FOLDER_NAME = "Example Folder"++# Connect to our Letta server+client = Letta(token=LETTA_API_KEY)++# get an available embedding_config+embedding_configs = client.embedding_models.list()+embedding_config = embedding_configs[0]++# Check if the folder already exists+try:+ folder_id = client.folders.retrieve_by_name(FOLDER_NAME)++# We got an API error. Check if it's a 404, meaning the folder doesn't exist.+except ApiError as e:+ if e.status_code == 404:+ # Create a new folder+ folder = client.folders.create(+ name=FOLDER_NAME,+ description="This is an example folder",+ instructions="Use this data folder to see how Letta works.",+ )+ folder_id = folder.id+ else:+ raise e++except Exception as e:+ # Something else went wrong+ raise e+++#+# There's two ways to upload a file to a folder.+#+# 1. From an existing file+# 2. From a string by encoding it into a base64 string+#+#++# 1. From an existing file+# "rb" means "read binary"+file = open("example-on-disk.txt", "rb")++# Upload the file to the folder+file = client.folders.files.upload(+ folder_id=folder_id,+ file=file,+ duplicate_handling="skip"+)++# 2. From a string by encoding it into a base64 string+import io++content = """+This is an example file. If you can read this,+the password is 'letta'.+"""++# Encode the string into bytes, and then create a file-like object+# that exists only in memory.+file_object = io.BytesIO(content.encode("utf-8"))++# Set the name of the file+file_object.name = "example.txt"++# Upload the file to the folder+file = client.folders.files.upload(+ folder_id=folder_id,+ file=file_object,+ duplicate_handling="skip"+)++#+# You can also upload PDFs!+# Letta extracts text from PDFs using OCR.+#++# Download the PDF to the local directory if it doesn't exist+if not os.path.exists("memgpt.pdf"):+ # Download the PDF+ print("Downloading memgpt.pdf")+ response = requests.get("https://arxiv.org/pdf/2310.08560")+ with open("memgpt.pdf", "wb") as f:+ f.write(response.content)++# Upload the PDF to the folder+file = client.folders.files.upload(+ folder_id=folder_id,+ file=open("memgpt.pdf", "rb"),+ duplicate_handling="skip"+)++#+# Now we need to create an agent that can use this folder+#++# Create an agent+agent = client.agents.create(+ model="openai/gpt-4o-mini",+ name="Example Agent",+ description="This agent looks at files and answers questions about them.",+ memory_blocks = [+ {+ "label": "human",+ "value": "The human wants to know about the files."+ },+ {+ "label": "persona",+ "value": "My name is Clippy, I answer questions about files."+ }+ ]+)++# Attach the data folder to the agent.+# Once the folder is attached, the agent will be able to see all+# files in the folder.+client.agents.folders.attach(+ agent_id=agent.id,+ folder_id=folder_id+)++########################################################+# This code makes a simple chatbot interface to the agent+########################################################++# Wrap this in a try/catch block to remove the agent in the event of an error+try:+ print(f"🤖 Connected to agent: {agent.name}")+ print("💡 Type 'quit' or 'exit' to end the conversation")+ print("=" * 50)++ while True:+ # Get user input+ try:+ user_input = input("\n👤 You: ").strip()+ except (EOFError, KeyboardInterrupt):+ print("\n👋 Goodbye!")+ break++ if user_input.lower() in ['quit', 'exit', 'q']:+ print("👋 Goodbye!")+ break++ if not user_input:+ continue++ # Stream the agent's response+ stream = client.agents.messages.create_stream(+ agent_id=agent.id,+ messages=[+ {+ "role": "user",+ "content": user_input+ }+ ],+ )++ for chunk in stream:+ print(chunk)++finally:+ client.agents.delete(agent.id)diff --git a/examples/mcp_example.py b/examples/mcp_example.pyindex 9352a1a37..25d1aaf81 100644--- a/examples/mcp_example.py+++ b/examples/mcp_example.py@@ -2,22 +2,33 @@ from pprint import pprintfrom letta_client import Letta+# Connect to Letta serverclient = Letta(base_url="http://localhost:8283")+# Use the "everything" mcp server:+# https://github.com/modelcontextprotocol/servers/tree/main/src/everythingmcp_server_name = "everything"mcp_tool_name = "echo"+# List all McpTool belonging to the "everything" mcp server.mcp_tools = client.tools.list_mcp_tools_by_server(mcp_server_name=mcp_server_name,)++# We can see that "echo" is one of the tools, but it's not+# a letta tool that can be added to a client (it has no tool id).for tool in mcp_tools:pprint(tool)+# Create a Tool (with a tool id) using the server and tool names.mcp_tool = client.tools.add_mcp_tool(mcp_server_name=mcp_server_name,mcp_tool_name=mcp_tool_name)+# Create an agent with the tool, using tool.id -- note that+# this is the ONLY tool in the agent, you typically want to+# also include the default tools.agent = client.agents.create(memory_blocks=[{@@ -31,6 +42,7 @@ agent = client.agents.create()print(f"Created agent id {agent.id}")+# Ask the agent to call the tool.response = client.agents.messages.create(agent_id=agent.id,messages=[diff --git a/examples/notebooks/Customizing memory management.ipynb b/examples/notebooks/Customizing memory management.ipynbindex af167dce3..2df343b4d 100644--- a/examples/notebooks/Customizing memory management.ipynb+++ b/examples/notebooks/Customizing memory management.ipynb@@ -253,15 +253,18 @@},{"cell_type": "code",- "execution_count": 11,+ "execution_count": null,"id": "7808912f-831b-4cdc-8606-40052eb809b4","metadata": {},"outputs": [],"source": [- "from typing import Optional, List\n",+ "from typing import Optional, List, TYPE_CHECKING\n","import json\n","\n",- "def task_queue_push(self: \"Agent\", task_description: str):\n",+ "if TYPE_CHECKING:\n",+ " from letta import AgentState\n",+ "\n",+ "def task_queue_push(agent_state: \"AgentState\", task_description: str):\n"," \"\"\"\n"," Push to a task queue stored in core memory. \n","\n",@@ -273,12 +276,12 @@" does not produce a response.\n"," \"\"\"\n"," import json\n",- " tasks = json.loads(self.memory.get_block(\"tasks\").value)\n",+ " tasks = json.loads(agent_state.memory.get_block(\"tasks\").value)\n"," tasks.append(task_description)\n",- " self.memory.update_block_value(\"tasks\", json.dumps(tasks))\n",+ " agent_state.memory.update_block_value(\"tasks\", json.dumps(tasks))\n"," return None\n","\n",- "def task_queue_pop(self: \"Agent\"):\n",+ "def task_queue_pop(agent_state: \"AgentState\"):\n"," \"\"\"\n"," Get the next task from the task queue \n","\n",@@ -288,12 +291,12 @@" None (the task queue is empty)\n"," \"\"\"\n"," import json\n",- " tasks = json.loads(self.memory.get_block(\"tasks\").value)\n",+ " tasks = json.loads(agent_state.memory.get_block(\"tasks\").value)\n"," if len(tasks) == 0: \n"," return None\n"," task = tasks[0]\n"," print(\"CURRENT TASKS: \", tasks)\n",- " self.memory.update_block_value(\"tasks\", json.dumps(tasks[1:]))\n",+ " agent_state.memory.update_block_value(\"tasks\", json.dumps(tasks[1:]))\n"," return task\n","\n","push_task_tool = client.tools.upsert_from_function(func=task_queue_push)\n",@@ -310,7 +313,7 @@},{"cell_type": "code",- "execution_count": 12,+ "execution_count": null,"id": "135fcf3e-59c4-4da3-b86b-dbffb21aa343","metadata": {},"outputs": [],@@ -336,10 +339,12 @@" ),\n"," CreateBlock(\n"," label=\"tasks\",\n",- " value=\"\",\n",+ " value=\"[]\",\n"," ),\n"," ],\n"," tool_ids=[push_task_tool.id, pop_task_tool.id],\n",+ " model=\"letta/letta-free\",\n",+ " embedding=\"letta/letta-free\",\n",")"]},diff --git a/letta/__main__.py b/letta/__main__.pydeleted file mode 100644index 89f11424b..000000000--- a/letta/__main__.py+++ /dev/null@@ -1,3 +0,0 @@-from .main import app--app()diff --git a/letta/functions/function_sets/base.py b/letta/functions/function_sets/base.pyindex 233ce7a51..36ebec986 100644--- a/letta/functions/function_sets/base.py+++ b/letta/functions/function_sets/base.py@@ -46,7 +46,7 @@ def conversation_search(self: "Agent", query: str, page: Optional[int] = 0) -> Ocount = RETRIEVAL_QUERY_DEFAULT_PAGE_SIZE# TODO: add paging by page number. currently cursor only works with strings.# original: start=page * count- messages = self.message_manager.list_user_messages_for_agent(+ messages = self.message_manager.list_messages_for_agent(agent_id=self.agent_state.id,actor=self.user,query_text=query,diff --git a/letta/llm_api/anthropic_client.py b/letta/llm_api/anthropic_client.pyindex 5a3206058..7b886dfca 100644--- a/letta/llm_api/anthropic_client.py+++ b/letta/llm_api/anthropic_client.py@@ -242,7 +242,8 @@ class AnthropicClient(LLMClientBase):# Move 'system' to the top levelif messages[0].role != "system":raise RuntimeError(f"First message is not a system message, instead has role {messages[0].role}")- data["system"] = messages[0].content if isinstance(messages[0].content, str) else messages[0].content[0].text+ system_content = messages[0].content if isinstance(messages[0].content, str) else messages[0].content[0].text+ data["system"] = self._add_cache_control_to_system_message(system_content)data["messages"] = [m.to_anthropic_dict(inner_thoughts_xml_tag=inner_thoughts_xml_tag,@@ -499,6 +500,22 @@ class AnthropicClient(LLMClientBase):return chat_completion_response+ def _add_cache_control_to_system_message(self, system_content):+ """Add cache control to system message content"""+ if isinstance(system_content, str):+ # For string content, convert to list format with cache control+ return [{"type": "text", "text": system_content, "cache_control": {"type": "ephemeral"}}]+ elif isinstance(system_content, list):+ # For list content, add cache control to the last text block+ cached_content = system_content.copy()+ for i in range(len(cached_content) - 1, -1, -1):+ if cached_content[i].get("type") == "text":+ cached_content[i]["cache_control"] = {"type": "ephemeral"}+ break+ return cached_content++ return system_content+def convert_tools_to_anthropic_format(tools: List[OpenAITool]) -> List[dict]:"""See: https://docs.anthropic.com/claude/docs/tool-usediff --git a/letta/schemas/llm_config.py b/letta/schemas/llm_config.pyindex 6d6b6039b..6259423fb 100644--- a/letta/schemas/llm_config.py+++ b/letta/schemas/llm_config.py@@ -63,7 +63,8 @@ class LLMConfig(BaseModel):description="The reasoning effort to use when generating text reasoning models",)max_reasoning_tokens: int = Field(- 0, description="Configurable thinking budget for extended thinking, only used if enable_reasoner is True. Minimum value is 1024."+ 0,+ description="Configurable thinking budget for extended thinking. Used for enable_reasoner and also for Google Vertex models like Gemini 2.5 Flash. Minimum value is 1024 when used with enable_reasoner.",)frequency_penalty: Optional[float] = Field(None, # Can also deafult to 0.0?diff --git a/letta/schemas/providers/ollama.py b/letta/schemas/providers/ollama.pyindex 8cc8f7202..df7d83468 100644--- a/letta/schemas/providers/ollama.py+++ b/letta/schemas/providers/ollama.py@@ -1,7 +1,6 @@from typing import Literalimport aiohttp-import requestsfrom pydantic import Fieldfrom letta.constants import DEFAULT_EMBEDDING_CHUNK_SIZE@@ -43,7 +42,7 @@ class OllamaProvider(OpenAIProvider):configs = []for model in response_json["models"]:- context_window = self.get_model_context_window(model["name"])+ context_window = await self._get_model_context_window(model["name"])if context_window is None:print(f"Ollama model {model['name']} has no context window, using default 32000")context_window = 32000@@ -92,63 +91,50 @@ class OllamaProvider(OpenAIProvider):)return configs- def get_model_context_window(self, model_name: str) -> int | None:- """Gets model context window for Ollama. As this can look different based on models,- we use the following for guidance:-- "llama.context_length": 8192,- "llama.embedding_length": 4096,- source: https://github.com/ollama/ollama/blob/main/docs/api.md#show-model-information-- FROM 2024-10-08- Notes from vLLM around keys- source: https://github.com/vllm-project/vllm/blob/72ad2735823e23b4e1cc79b7c73c3a5f3c093ab0/vllm/config.py#L3488-- possible_keys = [- # OPT- "max_position_embeddings",- # GPT-2- "n_positions",- # MPT- "max_seq_len",- # ChatGLM2- "seq_length",- # Command-R- "model_max_length",- # Whisper- "max_target_positions",- # Others- "max_sequence_length",- "max_seq_length",- "seq_len",- ]- max_position_embeddings- parse model cards: nous, dolphon, llama- """+ async def _get_model_context_window(self, model_name: str) -> int | None:endpoint = f"{self.base_url}/api/show"- payload = {"name": model_name, "verbose": True}- response = requests.post(endpoint, json=payload)- if response.status_code != 200:- return None+ payload = {"name": model_name}try:- model_info = response.json()- # Try to extract context window from model parameters- if "model_info" in model_info and "llama.context_length" in model_info["model_info"]:- return int(model_info["model_info"]["llama.context_length"])- except Exception:- pass- logger.warning(f"Failed to get model context window for {model_name}")+ async with aiohttp.ClientSession() as session:+ async with session.post(endpoint, json=payload) as response:+ if response.status != 200:+ error_text = await response.text()+ logger.warning(f"Failed to get model info for {model_name}: {response.status} - {error_text}")+ return None++ response_json = await response.json()+ model_info = response_json.get("model_info", {})++ if architecture := model_info.get("general.architecture"):+ if context_length := model_info.get(f"{architecture}.context_length"):+ return int(context_length)++ except Exception as e:+ logger.warning(f"Failed to get model context window for {model_name} with error: {e}")+return None- async def _get_model_embedding_dim_async(self, model_name: str):- async with aiohttp.ClientSession() as session:- async with session.post(f"{self.base_url}/api/show", json={"name": model_name, "verbose": True}) as response:- response_json = await response.json()+ async def _get_model_embedding_dim(self, model_name: str) -> int | None:+ endpoint = f"{self.base_url}/api/show"+ payload = {"name": model_name}++ try:+ async with aiohttp.ClientSession() as session:+ async with session.post(endpoint, json=payload) as response:+ if response.status != 200:+ error_text = await response.text()+ logger.warning(f"Failed to get model info for {model_name}: {response.status} - {error_text}")+ return None- if "model_info" not in response_json:- if "error" in response_json:- logger.warning("Ollama fetch model info error for %s: %s", model_name, response_json["error"])- return None+ response_json = await response.json()+ model_info = response_json.get("model_info", {})- return response_json["model_info"].get("embedding_length")+ if architecture := model_info.get("general.architecture"):+ if embedding_length := model_info.get(f"{architecture}.embedding_length"):+ return int(embedding_length)++ except Exception as e:+ logger.warning(f"Failed to get model embedding dimension for {model_name} with error: {e}")++ return Nonediff --git a/tests/test_base_functions.py b/tests/test_base_functions.pyindex f267c61dc..7ba28ee90 100644--- a/tests/test_base_functions.py+++ b/tests/test_base_functions.py@@ -151,6 +151,7 @@ def test_archival(agent_obj):def test_recall(server, agent_obj, default_user):"""Test that an agent can recall messages using a keyword via conversation search."""keyword = "banana"+ "".join(reversed(keyword))# Send messagesfor msg in ["hello", keyword, "tell me a fun fact"]: