Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

chore: merge oss (#3712)

Sarah Woodersbde471429414
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index ff63f2ac0..4c6269403 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -11,20 +11,25 @@ assignees: ''
A clear and concise description of what the bug is.
**Please describe your setup**
-- [ ] How did you install letta?
- - `pip install letta`? `pip install letta-nightly`? `git clone`?
+- [ ] How are you running Letta?
+ - Docker
+ - pip (legacy)
+ - From source
+ - Desktop
- [ ] Describe your setup
- What's your OS (Windows/MacOS/Linux)?
- - How are you running `letta`? (`cmd.exe`/Powershell/Anaconda Shell/Terminal)
+ - What is your `docker run ...` command (if applicable)
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Additional context**
Add any other context about the problem here.
+- What model you are using
+
+**Agent File (optional)**
+Please attach your `.af` file, as this helps with reproducing issues.
-**Letta Config**
-Please attach your `~/.letta/config` file or copy paste it below.
---
diff --git a/.github/scripts/model-sweep/conftest.py b/.github/scripts/model-sweep/conftest.py
new file mode 100644
index 000000000..db66a444c
--- /dev/null
+++ b/.github/scripts/model-sweep/conftest.py
@@ -0,0 +1,286 @@
+import logging
+import os
+import socket
+import threading
+import time
+from datetime import datetime, timezone
+from typing import Generator
+
+import pytest
+import requests
+from anthropic.types.beta.messages import BetaMessageBatch, BetaMessageBatchRequestCounts
+from dotenv import load_dotenv
+from letta_client import AsyncLetta, Letta
+
+from letta.schemas.agent import AgentState
+from letta.schemas.llm_config import LLMConfig
+from letta.services.organization_manager import OrganizationManager
+from letta.services.user_manager import UserManager
+from letta.settings import tool_settings
+
+
+def pytest_configure(config):
+ logging.basicConfig(level=logging.DEBUG)
+
+
+def disable_e2b_api_key() -> Generator[None, None, None]:
+ """
+ Temporarily disables the E2B API key by setting `tool_settings.e2b_api_key` to None
+ for the duration of the test. Restores the original value afterward.
+ """
+ from letta.settings import tool_settings
+
+ original_api_key = tool_settings.e2b_api_key
+ tool_settings.e2b_api_key = None
+ yield
+ tool_settings.e2b_api_key = original_api_key
+
+
+def check_e2b_key_is_set():
+ from letta.settings import tool_settings
+
+ original_api_key = tool_settings.e2b_api_key
+ assert original_api_key is not None, "Missing e2b key! Cannot execute these tests."
+ yield
+
+
+def default_organization():
+ """Fixture to create and return the default organization."""
+ manager = OrganizationManager()
+ org = manager.create_default_organization()
+ yield org
+
+
+def default_user(default_organization):
+ """Fixture to create and return the default user within the default organization."""
+ manager = UserManager()
+ user = manager.create_default_user(org_id=default_organization.id)
+ yield user
+
+
+def check_composio_key_set():
+ original_api_key = tool_settings.composio_api_key
+ assert original_api_key is not None, "Missing composio key! Cannot execute this test."
+ yield
+
+
+# --- Tool Fixtures ---
+def weather_tool_func():
+ def get_weather(location: str) -> str:
+ """
+ Fetches the current weather for a given location.
+
+ Parameters:
+ location (str): The location to get the weather for.
+
+ Returns:
+ str: A formatted string describing the weather in the given location.
+
+ Raises:
+ RuntimeError: If the request to fetch weather data fails.
+ """
+ import requests
+
+ url = f"https://wttr.in/{location}?format=%C+%t"
+
+ response = requests.get(url)
+ if response.status_code == 200:
+ weather_data = response.text
+ return f"The weather in {location} is {weather_data}."
+ else:
+ raise RuntimeError(f"Failed to get weather data, status code: {response.status_code}")
+
+ yield get_weather
+
+
+def print_tool_func():
+ """Fixture to create a tool with default settings and clean up after the test."""
+
+ def print_tool(message: str):
+ """
+ Args:
+ message (str): The message to print.
+
+ Returns:
+ str: The message that was printed.
+ """
+ print(message)
+ return message
+
+ yield print_tool
+
+
+def roll_dice_tool_func():
+ def roll_dice():
+ """
+ Rolls a 6 sided die.
+
+ Returns:
+ str: The roll result.
+ """
+ import time
+
+ time.sleep(1)
+ return "Rolled a 10!"
+
+ yield roll_dice
+
+
+def dummy_beta_message_batch() -> BetaMessageBatch:
+ return BetaMessageBatch(
+ id="msgbatch_013Zva2CMHLNnXjNJJKqJ2EF",
+ archived_at=datetime(2024, 8, 20, 18, 37, 24, 100435, tzinfo=timezone.utc),
+ cancel_initiated_at=datetime(2024, 8, 20, 18, 37, 24, 100435, tzinfo=timezone.utc),
+ created_at=datetime(2024, 8, 20, 18, 37, 24, 100435, tzinfo=timezone.utc),
+ ended_at=datetime(2024, 8, 20, 18, 37, 24, 100435, tzinfo=timezone.utc),
+ expires_at=datetime(2024, 8, 20, 18, 37, 24, 100435, tzinfo=timezone.utc),
+ processing_status="in_progress",
+ request_counts=BetaMessageBatchRequestCounts(
+ canceled=10,
+ errored=30,
+ expired=10,
+ processing=100,
+ succeeded=50,
+ ),
+ results_url="https://api.anthropic.com/v1/messages/batches/msgbatch_013Zva2CMHLNnXjNJJKqJ2EF/results",
+ type="message_batch",
+ )
+
+
+# --- Model Sweep ---
+# Global flag to track server state
+_server_started = False
+_server_url = None
+
+
+def _start_server_once() -> str:
+ """Start server exactly once, return URL"""
+ global _server_started, _server_url
+
+ if _server_started and _server_url:
+ return _server_url
+
+ url = os.getenv("LETTA_SERVER_URL", "http://localhost:8283")
+
+ # Check if already running
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+ if s.connect_ex(("localhost", 8283)) == 0:
+ _server_started = True
+ _server_url = url
+ return url
+
+ # Start server (your existing logic)
+ if not os.getenv("LETTA_SERVER_URL"):
+
+ def _run_server():
+ load_dotenv()
+ from letta.server.rest_api.app import start_server
+
+ start_server(debug=True)
+
+ thread = threading.Thread(target=_run_server, daemon=True)
+ thread.start()
+
+ # Poll until up
+ timeout_seconds = 30
+ deadline = time.time() + timeout_seconds
+ while time.time() < deadline:
+ try:
+ resp = requests.get(url + "/v1/health")
+ if resp.status_code < 500:
+ break
+ except requests.exceptions.RequestException:
+ pass
+ time.sleep(0.1)
+ else:
+ raise RuntimeError(f"Could not reach {url} within {timeout_seconds}s")
+
+ _server_started = True
+ _server_url = url
+ return url
+
+
+# ------------------------------
+# Fixtures
+# ------------------------------
+
+
[email protected](scope="module")
+def server_url() -> str:
+ """Return URL of already-started server"""
+ return _start_server_once()
+
+
[email protected](scope="module")
+def client(server_url: str) -> Letta:
+ """
+ Creates and returns a synchronous Letta REST client for testing.
+ """
+ client_instance = Letta(base_url=server_url)
+ yield client_instance
+
+
[email protected](scope="function")
+def async_client(server_url: str) -> AsyncLetta:
+ """
+ Creates and returns an asynchronous Letta REST client for testing.
+ """
+ async_client_instance = AsyncLetta(base_url=server_url)
+ yield async_client_instance
+
+
[email protected](scope="module")
+def agent_state(client: Letta) -> AgentState:
+ """
+ Creates and returns an agent state for testing with a pre-configured agent.
+ The agent is named 'supervisor' and is configured with base tools and the roll_dice tool.
+ """
+ client.tools.upsert_base_tools()
+
+ send_message_tool = client.tools.list(name="send_message")[0]
+ agent_state_instance = client.agents.create(
+ name="supervisor",
+ include_base_tools=False,
+ tool_ids=[send_message_tool.id],
+ model="openai/gpt-4o",
+ embedding="letta/letta-free",
+ tags=["supervisor"],
+ )
+ yield agent_state_instance
+
+ client.agents.delete(agent_state_instance.id)
+
+
[email protected](scope="module")
+def all_available_llm_configs(client: Letta) -> [LLMConfig]:
+ """
+ Returns a list of all available LLM configs.
+ """
+ llm_configs = client.models.list()
+ return llm_configs
+
+
+# create a client to the started server started at
+def get_available_llm_configs() -> [LLMConfig]:
+ """Get configs, starting server if needed"""
+ server_url = _start_server_once()
+ temp_client = Letta(base_url=server_url)
+ return temp_client.models.list()
+
+
+# dynamically insert llm_config paramter at collection time
+def pytest_generate_tests(metafunc):
+ """Dynamically parametrize tests that need llm_config."""
+ if "llm_config" in metafunc.fixturenames:
+ configs = get_available_llm_configs()
+ if configs:
+ metafunc.parametrize("llm_config", configs, ids=[c.model for c in configs])
diff --git a/.github/scripts/model-sweep/feature_mappings.json b/.github/scripts/model-sweep/feature_mappings.json
new file mode 100644
index 000000000..41222e154
--- /dev/null
+++ b/.github/scripts/model-sweep/feature_mappings.json
@@ -0,0 +1,21 @@
+{
+ "Basic": [
+ "test_greeting_with_assistant_message",
+ "test_greeting_without_assistant_message",
+ "test_async_greeting_with_assistant_message",
+ "test_agent_loop_error",
+ "test_step_stream_agent_loop_error",
+ "test_step_streaming_greeting_with_assistant_message",
+ "test_step_streaming_greeting_without_assistant_message",
+ "test_step_streaming_tool_call",
+ "test_tool_call",
+ "test_auto_summarize"
+ ],
+ "Token Streaming": [
+ "test_token_streaming_greeting_with_assistant_message",
+ "test_token_streaming_greeting_without_assistant_message",
+ "test_token_streaming_agent_loop_error",
+ "test_token_streaming_tool_call"
+ ],
+ "Multimodal": ["test_base64_image_input", "test_url_image_input"]
+}
diff --git a/.github/scripts/model-sweep/generate_model_sweep_markdown.py b/.github/scripts/model-sweep/generate_model_sweep_markdown.py
new file mode 100644
index 000000000..38552a8c1
--- /dev/null
+++ b/.github/scripts/model-sweep/generate_model_sweep_markdown.py
@@ -0,0 +1,495 @@
+#!/usr/bin/env python3
+import json
+import os
+import sys
+from collections import defaultdict
+from datetime import datetime
+
+
+def load_feature_mappings(config_file=None):
+ """Load feature mappings from config file."""
+ if config_file is None:
+ # Default to feature_mappings.json in the same directory as this script
+ script_dir = os.path.dirname(os.path.abspath(__file__))
+ config_file = os.path.join(script_dir, "feature_mappings.json")
+
+ try:
+ with open(config_file, "r") as f:
+ return json.load(f)
+ except FileNotFoundError:
+ print(f"Error: Could not find feature mappings config file '{config_file}'")
+ sys.exit(1)
+ except json.JSONDecodeError:
+ print(f"Error: Invalid JSON in feature mappings config file '{config_file}'")
+ sys.exit(1)
+
+
+def get_support_status(passed_tests, feature_tests):
+ """Determine support status for a feature category."""
+ if not feature_tests:
+ return "❓" # Unknown - no tests for this feature
+
+ # Filter out error tests when checking for support
+ non_error_tests = [test for test in feature_tests if not test.endswith("_error")]
+ error_tests = [test for test in feature_tests if test.endswith("_error")]
+
+ # Check which non-error tests passed
+ passed_non_error_tests = [test for test in non_error_tests if test in passed_tests]
+
+ # If there are no non-error tests, only error tests, treat as unknown
+ if not non_error_tests:
+ return "❓" # Only error tests available
+
+ # Support is based only on non-error tests
+ if len(passed_non_error_tests) == len(non_error_tests):
+ return "✅" # Full support
+ elif len(passed_non_error_tests) == 0:
+ return "❌" # No support
+ else:
+ return "⚠️" # Partial support
+
+
+def categorize_tests(all_test_names, feature_mapping):
+ """Categorize test names into feature buckets."""
+ categorized = {feature: [] for feature in feature_mapping.keys()}
+
+ for test_name in all_test_names:
+ for feature, test_patterns in feature_mapping.items():
+ if test_name in test_patterns:
+ categorized[feature].append(test_name)
+ break
+
+ return categorized
+
+
+def calculate_support_score(feature_support, feature_order):
+ """Calculate a numeric support score for ranking models.
+
+ For partial support, the score is weighted by the position of the feature
+ in the feature_order list (earlier features get higher weight).
+ """
+ score = 0
+ max_features = len(feature_order)
+
+ for feature, status in feature_support.items():
+ # Get position weight (earlier features get higher weight)
+ if feature in feature_order:
+ position_weight = (max_features - feature_order.index(feature)) / max_features
+ else:
+ position_weight = 0.5 # Default weight for unmapped features
+
+ if status == "✅": # Full support
+ score += 10 * position_weight
+ elif status == "⚠️": # Partial support - weighted by column position
+ score += 5 * position_weight
+ elif status == "❌": # No support
+ score += 1 * position_weight
+ # Unknown (❓) gets 0 points
+ return score
+
+
+def calculate_provider_support_score(models_data, feature_order):
+ """Calculate a provider-level support score based on all models' support scores."""
+ if not models_data:
+ return 0
+
+ # Calculate the average support score across all models in the provider
+ total_score = sum(model["support_score"] for model in models_data)
+ return total_score / len(models_data)
+
+
+def get_test_function_line_numbers(test_file_path):
+ """Extract line numbers for test functions from the test file."""
+ test_line_numbers = {}
+
+ try:
+ with open(test_file_path, "r") as f:
+ lines = f.readlines()
+
+ for i, line in enumerate(lines, 1):
+ if "def test_" in line and line.strip().startswith("def test_"):
+ # Extract function name
+ func_name = line.strip().split("def ")[1].split("(")[0]
+ test_line_numbers[func_name] = i
+ except FileNotFoundError:
+ print(f"Warning: Could not find test file at {test_file_path}")
+
+ return test_line_numbers
+
+
+def get_github_repo_info():
+ """Get GitHub repository information from git remote."""
+ try:
+ # Try to get the GitHub repo URL from git remote
+ import subprocess
+
+ result = subprocess.run(["git", "remote", "get-url", "origin"], capture_output=True, text=True, cwd=os.path.dirname(__file__))
+ if result.returncode == 0:
+ remote_url = result.stdout.strip()
+ # Parse GitHub URL
+ if "github.com" in remote_url:
+ if remote_url.startswith("https://"):
+ # https://github.com/user/repo.git -> user/repo
+ repo_path = remote_url.replace("https://github.com/", "").replace(".git", "")
+ elif remote_url.startswith("git@"):
+ # [email protected]:user/repo.git -> user/repo
+ repo_path = remote_url.split(":")[1].replace(".git", "")
+ else:
+ return None
+ return repo_path
+ except:
+ pass
+
+ # Default fallback
+ return "letta-ai/letta"
+
+
+def generate_test_details(model_info, feature_mapping):
+ """Generate detailed test results for a model."""
+ details = []
+
+ # Get test function line numbers
+ script_dir = os.path.dirname(os.path.abspath(__file__))
+ test_file_path = os.path.join(script_dir, "model_sweep.py")
+ test_line_numbers = get_test_function_line_numbers(test_file_path)
+
+ # Use the main branch GitHub URL
+ base_github_url = "https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py"
+
+ for feature, tests in model_info["categorized_tests"].items():
+ if not tests:
+ continue
+
+ details.append(f"### {feature}")
+ details.append("")
+
+ for test in sorted(tests):
+ if test in model_info["passed_tests"]:
+ status = "✅"
+ elif test in model_info["failed_tests"]:
+ status = "❌"
+ else:
+ status = "❓"
+
+ # Create GitHub link if we have line number info
+ if test in test_line_numbers:
+ line_num = test_line_numbers[test]
+ github_link = f"{base_github_url}#L{line_num}"
+ details.append(f"- {status} [`{test}`]({github_link})")
+ else:
+ details.append(f"- {status} `{test}`")
+ details.append("")
+
+ return details
+
+
+def calculate_column_widths(all_provider_data, feature_mapping):
+ """Calculate the maximum width needed for each column across all providers."""
+ widths = {"model": len("Model"), "context_window": len("Context Window"), "last_scanned": len("Last Scanned")}
+
+ # Feature column widths
+ for feature in feature_mapping.keys():
+ widths[feature] = len(feature)
+
+ # Check all model data for maximum widths
+ for provider_data in all_provider_data.values():
+ for model_info in provider_data:
+ # Model name width (including backticks)
+ model_width = len(f"`{model_info['name']}`")
+ widths["model"] = max(widths["model"], model_width)
+
+ # Context window width (with commas)
+ context_width = len(f"{model_info['context_window']:,}")
+ widths["context_window"] = max(widths["context_window"], context_width)
+
+ # Last scanned width
+ widths["last_scanned"] = max(widths["last_scanned"], len(str(model_info["last_scanned"])))
+
+ # Feature support symbols are always 2 chars, so no need to check
+
+ return widths
+
+
+def process_model_sweep_report(input_file, output_file, config_file=None, debug=False):
+ """Convert model sweep JSON data to MDX report."""
+
+ # Load feature mappings from config file
+ feature_mapping = load_feature_mappings(config_file)
+
+ # if debug:
+ # print("DEBUG: Feature mappings loaded:")
+ # for feature, tests in feature_mapping.items():
+ # print(f" {feature}: {tests}")
+ # print()
+
+ # Read the JSON data
+ with open(input_file, "r") as f:
+ data = json.load(f)
+
+ tests = data.get("tests", [])
+
+ # if debug:
+ # print("DEBUG: Tests loaded:")
+ # print([test['outcome'] for test in tests if 'haiku' in test['nodeid']])
+
+ # Calculate summary statistics
+ providers = set(test["metadata"]["llm_config"]["provider_name"] for test in tests)
+ models = set(test["metadata"]["llm_config"]["model"] for test in tests)
+ total_tests = len(tests)
+
+ # Start building the MDX
+ mdx_lines = [
+ "---",
+ "title: Support Models",
+ f"generated: {datetime.now().isoformat()}",
+ "---",
+ "",
+ "# Supported Models",
+ "",
+ "## Overview",
+ "",
+ "Letta routinely runs automated scans against available providers and models. These are the results of the latest scan.",
+ "",
+ f"Ran {total_tests} tests against {len(models)} models across {len(providers)} providers on {datetime.now().strftime('%B %dth, %Y')}",
+ "",
+ "",
+ ]
+
+ # Group tests by provider
+ provider_groups = defaultdict(list)
+ for test in tests:
+ provider_name = test["metadata"]["llm_config"]["provider_name"]
+ provider_groups[provider_name].append(test)
+
+ # Process all providers first to collect model data
+ all_provider_data = {}
+ provider_support_scores = {}
+
+ for provider_name in provider_groups.keys():
+ provider_tests = provider_groups[provider_name]
+
+ # Group tests by model within this provider
+ model_groups = defaultdict(list)
+ for test in provider_tests:
+ model_name = test["metadata"]["llm_config"]["model"]
+ model_groups[model_name].append(test)
+
+ # Process all models to calculate support scores for ranking
+ model_data = []
+ for model_name in model_groups.keys():
+ model_tests = model_groups[model_name]
+
+ # if debug:
+ # print(f"DEBUG: Processing model '{model_name}' in provider '{provider_name}'")
+
+ # Extract unique test names for passed and failed tests
+ passed_tests = set()
+ failed_tests = set()
+ all_test_names = set()
+
+ for test in model_tests:
+ # Extract test name from nodeid (split on :: and [)
+ test_name = test["nodeid"].split("::")[1].split("[")[0]
+ all_test_names.add(test_name)
+
+ # if debug:
+ # print(f" Test name: {test_name}")
+ # print(f" Outcome: {test}")
+ if test["outcome"] == "passed":
+ passed_tests.add(test_name)
+ elif test["outcome"] == "failed":
+ failed_tests.add(test_name)
+
+ # if debug:
+ # print(f" All test names found: {sorted(all_test_names)}")
+ # print(f" Passed tests: {sorted(passed_tests)}")
+ # print(f" Failed tests: {sorted(failed_tests)}")
+
+ # Categorize tests into features
+ categorized_tests = categorize_tests(all_test_names, feature_mapping)
+
+ # if debug:
+ # print(f" Categorized tests:")
+ # for feature, tests in categorized_tests.items():
+ # print(f" {feature}: {tests}")
+
+ # Determine support status for each feature
+ feature_support = {}
+ for feature_name in feature_mapping.keys():
+ feature_support[feature_name] = get_support_status(passed_tests, categorized_tests[feature_name])
+
+ # if debug:
+ # print(f" Feature support:")
+ # for feature, status in feature_support.items():
+ # print(f" {feature}: {status}")
+ # print()
+
+ # Get context window and last scanned time
+ context_window = model_tests[0]["metadata"]["llm_config"]["context_window"]
+
+ # Try to get time_last_scanned from metadata, fallback to current time
+ try:
+ last_scanned = model_tests[0]["metadata"].get(
+ "time_last_scanned", model_tests[0]["metadata"].get("timestamp", datetime.now().isoformat())
+ )
+ # Format timestamp if it's a full ISO string
+ if "T" in str(last_scanned):
+ last_scanned = str(last_scanned).split("T")[0] # Just the date part
+ except:
+ last_scanned = "Unknown"
+
+ # Calculate support score for ranking
+ feature_order = list(feature_mapping.keys())
+ support_score = calculate_support_score(feature_support, feature_order)
+
+ # Store model data for sorting
+ model_data.append(
+ {
+ "name": model_name,
+ "feature_support": feature_support,
+ "context_window": context_window,
+ "last_scanned": last_scanned,
+ "support_score": support_score,
+ "failed_tests": failed_tests,
+ "passed_tests": passed_tests,
+ "categorized_tests": categorized_tests,
+ }
+ )
+
+ # Sort models by support score (descending) then by name (ascending)
+ model_data.sort(key=lambda x: (-x["support_score"], x["name"]))
+
+ # Store provider data
+ all_provider_data[provider_name] = model_data
+ provider_support_scores[provider_name] = calculate_provider_support_score(model_data, list(feature_mapping.keys()))
+
+ # Calculate column widths for consistent formatting (add details column)
+ column_widths = calculate_column_widths(all_provider_data, feature_mapping)
+ column_widths["details"] = len("Details")
+
+ # Sort providers by support score (descending) then by name (ascending)
+ sorted_providers = sorted(provider_support_scores.keys(), key=lambda x: (-provider_support_scores[x], x))
+
+ # Generate tables for all providers first
+ for provider_name in sorted_providers:
+ model_data = all_provider_data[provider_name]
+ support_score = provider_support_scores[provider_name]
+
+ # Create dynamic headers with proper padding and centering
+ feature_names = list(feature_mapping.keys())
+
+ # Build header row with left-aligned first column, centered others
+ header_parts = [f"{'Model':<{column_widths['model']}}"]
+ for feature in feature_names:
+ header_parts.append(f"{feature:^{column_widths[feature]}}")
+ header_parts.extend(
+ [
+ f"{'Context Window':^{column_widths['context_window']}}",
+ f"{'Last Scanned':^{column_widths['last_scanned']}}",
+ f"{'Details':^{column_widths['details']}}",
+ ]
+ )
+ header_row = "| " + " | ".join(header_parts) + " |"
+
+ # Build separator row with left-aligned first column, centered others
+ separator_parts = [f"{'-' * column_widths['model']}"]
+ for feature in feature_names:
+ separator_parts.append(f":{'-' * (column_widths[feature] - 2)}:")
+ separator_parts.extend(
+ [
+ f":{'-' * (column_widths['context_window'] - 2)}:",
+ f":{'-' * (column_widths['last_scanned'] - 2)}:",
+ f":{'-' * (column_widths['details'] - 2)}:",
+ ]
+ )
+ separator_row = "|" + "|".join(separator_parts) + "|"
+
+ # Add provider section without percentage
+ mdx_lines.extend([f"## {provider_name}", "", header_row, separator_row])
+
+ # Generate table rows for sorted models with proper padding
+ for model_info in model_data:
+ # Create anchor for model details
+ model_anchor = model_info["name"].replace("/", "_").replace(":", "_").replace("-", "_").lower()
+ details_anchor = f"{provider_name.lower().replace(' ', '_')}_{model_anchor}_details"
+
+ # Build row with left-aligned first column, centered others
+ row_parts = [f"`{model_info['name']}`".ljust(column_widths["model"])]
+ for feature in feature_names:
+ row_parts.append(f"{model_info['feature_support'][feature]:^{column_widths[feature]}}")
+ row_parts.extend(
+ [
+ f"{model_info['context_window']:,}".center(column_widths["context_window"]),
+ f"{model_info['last_scanned']}".center(column_widths["last_scanned"]),
+ f"[View](#{details_anchor})".center(column_widths["details"]),
+ ]
+ )
+ row = "| " + " | ".join(row_parts) + " |"
+ mdx_lines.append(row)
+
+ # Add spacing between provider tables
+ mdx_lines.extend(["", ""])
+
+ # Add detailed test results section after all tables
+ mdx_lines.extend(["---", "", "# Detailed Test Results", ""])
+
+ for provider_name in sorted_providers:
+ model_data = all_provider_data[provider_name]
+ mdx_lines.extend([f"## {provider_name}", ""])
+
+ for model_info in model_data:
+ model_anchor = model_info["name"].replace("/", "_").replace(":", "_").replace("-", "_").lower()
+ details_anchor = f"{provider_name.lower().replace(' ', '_')}_{model_anchor}_details"
+ mdx_lines.append(f'<a id="{details_anchor}"></a>')
+ mdx_lines.append(f"### {model_info['name']}")
+ mdx_lines.append("")
+
+ # Add test details
+ test_details = generate_test_details(model_info, feature_mapping)
+ mdx_lines.extend(test_details)
+
+ # Add spacing between providers in details section
+ mdx_lines.extend(["", ""])
+
+ # Write the MDX file
+ with open(output_file, "w") as f:
+ f.write("\n".join(mdx_lines))
+
+ print(f"Model sweep report saved to {output_file}")
+
+
+def main():
+ input_file = "model_sweep_report.json"
+ output_file = "model_sweep_report.mdx"
+ config_file = None
+ debug = False
+
+ # Allow command line arguments
+ if len(sys.argv) > 1:
+ # Use the file located in the same directory as this script
+ script_dir = os.path.dirname(os.path.abspath(__file__))
+ input_file = os.path.join(script_dir, sys.argv[1])
+ if len(sys.argv) > 2:
+ # Use the file located in the same directory as this script
+ script_dir = os.path.dirname(os.path.abspath(__file__))
+ output_file = os.path.join(script_dir, sys.argv[2])
+ if len(sys.argv) > 3:
+ config_file = sys.argv[3]
+ if len(sys.argv) > 4 and sys.argv[4] == "--debug":
+ debug = True
+
+ try:
+ process_model_sweep_report(input_file, output_file, config_file, debug)
+ except FileNotFoundError:
+ print(f"Error: Could not find input file '{input_file}'")
+ sys.exit(1)
+ except json.JSONDecodeError:
+ print(f"Error: Invalid JSON in file '{input_file}'")
+ sys.exit(1)
+ except Exception as e:
+ print(f"Error: {e}")
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.github/scripts/model-sweep/model_sweep.py b/.github/scripts/model-sweep/model_sweep.py
new file mode 100644
index 000000000..c92b93643
--- /dev/null
+++ b/.github/scripts/model-sweep/model_sweep.py
@@ -0,0 +1,786 @@
+import base64
+import json
+import os
+import socket
+import threading
+import time
+import uuid
+from typing import Any, Dict, List
+
+import httpx
+import pytest
+import requests
+from dotenv import load_dotenv
+from letta_client import Letta, MessageCreate, Run
+from letta_client.core.api_error import ApiError
+from letta_client.types import (
+ AssistantMessage,
+ Base64Image,
+ ImageContent,
+ LettaUsageStatistics,
+ ReasoningMessage,
+ TextContent,
+ ToolCallMessage,
+ ToolReturnMessage,
+ UrlImage,
+ UserMessage,
+)
+
+from letta.schemas.agent import AgentState
+from letta.schemas.llm_config import LLMConfig
+
+# ------------------------------
+# Helper Functions and Constants
+# ------------------------------
+
+
+def get_llm_config(filename: str, llm_config_dir: str = "tests/configs/llm_model_configs") -> LLMConfig:
+ filename = os.path.join(llm_config_dir, filename)
+ config_data = json.load(open(filename, "r"))
+ llm_config = LLMConfig(**config_data)
+ return llm_config
+
+
+def roll_dice(num_sides: int) -> int:
+ """
+ Returns a random number between 1 and num_sides.
+ Args:
+ num_sides (int): The number of sides on the die.
+ Returns:
+ int: A random integer between 1 and num_sides, representing the die roll.
+ """
+ import random
+
+ return random.randint(1, num_sides)
+
+
+USER_MESSAGE_OTID = str(uuid.uuid4())
+USER_MESSAGE_RESPONSE: str = "Teamwork makes the dream work"
+USER_MESSAGE_FORCE_REPLY: List[MessageCreate] = [
+ MessageCreate(
+ role="user",
+ content=f"This is an automated test message. Call the send_message tool with the message '{USER_MESSAGE_RESPONSE}'.",
+ otid=USER_MESSAGE_OTID,
+ )
+]
+USER_MESSAGE_ROLL_DICE: List[MessageCreate] = [
+ MessageCreate(
+ role="user",
+ content="This is an automated test message. Call the roll_dice tool with 16 sides and tell me the outcome.",
+ otid=USER_MESSAGE_OTID,
+ )
+]
+URL_IMAGE = "https://upload.wikimedia.org/wikipedia/commons/a/a7/Camponotus_flavomarginatus_ant.jpg"
+USER_MESSAGE_URL_IMAGE: List[MessageCreate] = [
+ MessageCreate(
+ role="user",
+ content=[
+ ImageContent(source=UrlImage(url=URL_IMAGE)),
+ TextContent(text="What is in this image?"),
+ ],
+ otid=USER_MESSAGE_OTID,
+ )
+]
+BASE64_IMAGE = base64.standard_b64encode(httpx.get(URL_IMAGE).content).decode("utf-8")
+USER_MESSAGE_BASE64_IMAGE: List[MessageCreate] = [
+ MessageCreate(
+ role="user",
+ content=[
+ ImageContent(source=Base64Image(data=BASE64_IMAGE, media_type="image/jpeg")),
+ TextContent(text="What is in this image?"),
+ ],
+ otid=USER_MESSAGE_OTID,
+ )
+]
+all_configs = [
+ "openai-gpt-4o-mini.json",
+ # "azure-gpt-4o-mini.json", # TODO: Re-enable on new agent loop
+ "claude-3-5-sonnet.json",
+ "claude-3-7-sonnet.json",
+ "claude-3-7-sonnet-extended.json",
+ "gemini-1.5-pro.json",
+ "gemini-2.5-flash-vertex.json",
+ "gemini-2.5-pro-vertex.json",
+ "together-qwen-2.5-72b-instruct.json",
+ "ollama.json",
+]
+requested = os.getenv("LLM_CONFIG_FILE")
+filenames = [requested] if requested else all_configs
+TESTED_LLM_CONFIGS: List[LLMConfig] = [get_llm_config(fn) for fn in filenames]
+
+
+def assert_greeting_with_assistant_message_response(
+ messages: List[Any],
+ streaming: bool = False,
+ token_streaming: bool = False,
+ from_db: bool = False,
+) -> None:
+ """
+ Asserts that the messages list follows the expected sequence:
+ ReasoningMessage -> AssistantMessage.
+ """
+ expected_message_count = 3 if streaming or from_db else 2
+ assert len(messages) == expected_message_count
+
+ index = 0
+ if from_db:
+ assert isinstance(messages[index], UserMessage)
+ assert messages[index].otid == USER_MESSAGE_OTID
+ index += 1
+
+ # Agent Step 1
+ assert isinstance(messages[index], ReasoningMessage)
+ assert messages[index].otid and messages[index].otid[-1] == "0"
+ index += 1
+
+ assert isinstance(messages[index], AssistantMessage)
+ if not token_streaming:
+ assert USER_MESSAGE_RESPONSE in messages[index].content
+ assert messages[index].otid and messages[index].otid[-1] == "1"
+ index += 1
+
+ if streaming:
+ assert isinstance(messages[index], LettaUsageStatistics)
+ assert messages[index].prompt_tokens > 0
+ assert messages[index].completion_tokens > 0
+ assert messages[index].total_tokens > 0
+ assert messages[index].step_count > 0
+
+
+def assert_greeting_without_assistant_message_response(
+ messages: List[Any],
+ streaming: bool = False,
+ token_streaming: bool = False,
+ from_db: bool = False,
+) -> None:
+ """
+ Asserts that the messages list follows the expected sequence:
+ ReasoningMessage -> ToolCallMessage -> ToolReturnMessage.
+ """
+ expected_message_count = 4 if streaming or from_db else 3
+ assert len(messages) == expected_message_count
+
+ index = 0
+ if from_db:
+ assert isinstance(messages[index], UserMessage)
+ assert messages[index].otid == USER_MESSAGE_OTID
+ index += 1
+
+ # Agent Step 1
+ assert isinstance(messages[index], ReasoningMessage)
+ assert messages[index].otid and messages[index].otid[-1] == "0"
+ index += 1
+
+ assert isinstance(messages[index], ToolCallMessage)
+ assert messages[index].tool_call.name == "send_message"
+ if not token_streaming:
+ assert USER_MESSAGE_RESPONSE in messages[index].tool_call.arguments
+ assert messages[index].otid and messages[index].otid[-1] == "1"
+ index += 1
+
+ # Agent Step 2
+ assert isinstance(messages[index], ToolReturnMessage)
+ assert messages[index].otid and messages[index].otid[-1] == "0"
+ index += 1
+
+ if streaming:
+ assert isinstance(messages[index], LettaUsageStatistics)
+
+
+def assert_tool_call_response(
+ messages: List[Any],
+ streaming: bool = False,
+ from_db: bool = False,
+) -> None:
+ """
+ Asserts that the messages list follows the expected sequence:
+ ReasoningMessage -> ToolCallMessage -> ToolReturnMessage ->
+ ReasoningMessage -> AssistantMessage.
+ """
+ expected_message_count = 6 if streaming else 7 if from_db else 5
+ assert len(messages) == expected_message_count
+
+ index = 0
+ if from_db:
+ assert isinstance(messages[index], UserMessage)
+ assert messages[index].otid == USER_MESSAGE_OTID
+ index += 1
+
+ # Agent Step 1
+ assert isinstance(messages[index], ReasoningMessage)
+ assert messages[index].otid and messages[index].otid[-1] == "0"
+ index += 1
+
+ assert isinstance(messages[index], ToolCallMessage)
+ assert messages[index].otid and messages[index].otid[-1] == "1"
+ index += 1
+
+ # Agent Step 2
+ assert isinstance(messages[index], ToolReturnMessage)
+ assert messages[index].otid and messages[index].otid[-1] == "0"
+ index += 1
+
+ # Hidden User Message
+ if from_db:
+ assert isinstance(messages[index], UserMessage)
+ assert "request_heartbeat=true" in messages[index].content
+ index += 1
+
+ # Agent Step 3
+ assert isinstance(messages[index], ReasoningMessage)
+ assert messages[index].otid and messages[index].otid[-1] == "0"
+ index += 1
+
+ assert isinstance(messages[index], AssistantMessage)
+ assert messages[index].otid and messages[index].otid[-1] == "1"
+ index += 1
+
+ if streaming:
+ assert isinstance(messages[index], LettaUsageStatistics)
+
+
+def assert_image_input_response(
+ messages: List[Any],
+ streaming: bool = False,
+ token_streaming: bool = False,
+ from_db: bool = False,
+) -> None:
+ """
+ Asserts that the messages list follows the expected sequence:
+ ReasoningMessage -> AssistantMessage.
+ """
+ expected_message_count = 3 if streaming or from_db else 2
+ assert len(messages) == expected_message_count
+
+ index = 0
+ if from_db:
+ assert isinstance(messages[index], UserMessage)
+ assert messages[index].otid == USER_MESSAGE_OTID
+ index += 1
+
+ # Agent Step 1
+ assert isinstance(messages[index], ReasoningMessage)
+ assert messages[index].otid and messages[index].otid[-1] == "0"
+ index += 1
+
+ assert isinstance(messages[index], AssistantMessage)
+ assert messages[index].otid and messages[index].otid[-1] == "1"
+ index += 1
+
+ if streaming:
+ assert isinstance(messages[index], LettaUsageStatistics)
+ assert messages[index].prompt_tokens > 0
+ assert messages[index].completion_tokens > 0
+ assert messages[index].total_tokens > 0
+ assert messages[index].step_count > 0
+
+
+def accumulate_chunks(chunks: List[Any]) -> List[Any]:
+ """
+ Accumulates chunks into a list of messages.
+ """
+ messages = []
+ current_message = None
+ prev_message_type = None
+ for chunk in chunks:
+ current_message_type = chunk.message_type
+ if prev_message_type != current_message_type:
+ messages.append(current_message)
+ current_message = None
+ if current_message is None:
+ current_message = chunk
+ else:
+ pass # TODO: actually accumulate the chunks. For now we only care about the count
+ prev_message_type = current_message_type
+ messages.append(current_message)
+ return [m for m in messages if m is not None]
+
+
+def wait_for_run_completion(client: Letta, run_id: str, timeout: float = 30.0, interval: float = 0.5) -> Run:
+ start = time.time()
+ while True:
+ run = client.runs.retrieve(run_id)
+ if run.status == "completed":
+ return run
+ if run.status == "failed":
+ raise RuntimeError(f"Run {run_id} did not complete: status = {run.status}")
+ if time.time() - start > timeout:
+ raise TimeoutError(f"Run {run_id} did not complete within {timeout} seconds (last status: {run.status})")
+ time.sleep(interval)
+
+
+def assert_tool_response_dict_messages(messages: List[Dict[str, Any]]) -> None:
+ """
+ Asserts that a list of message dictionaries contains the expected types and statuses.
+
+ Expected order:
+ 1. reasoning_message
+ 2. tool_call_message
+ 3. tool_return_message (with status 'success')
+ 4. reasoning_message
+ 5. assistant_message
+ """
+ assert isinstance(messages, list)
+ assert messages[0]["message_type"] == "reasoning_message"
+ assert messages[1]["message_type"] == "assistant_message"
+
+
+# ------------------------------
+# Test Cases
+# ------------------------------
+
+# def test_that_ci_workflow_works(
+# disable_e2b_api_key: Any,
+# client: Letta,
+# agent_state: AgentState,
+# llm_config: LLMConfig,
+# json_metadata: pytest.FixtureRequest,
+# ) -> None:
+# """
+# Tests that the CI workflow works.
+# """
+# json_metadata["test_type"] = "debug"
+
+
+def test_greeting_with_assistant_message(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ agent_state: AgentState,
+ llm_config: LLMConfig,
+ json_metadata: pytest.FixtureRequest,
+) -> None:
+ """
+ Tests sending a message with a synchronous client.
+ Verifies that the response messages follow the expected order.
+ """
+ json_metadata["llm_config"] = dict(llm_config)
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_FORCE_REPLY,
+ )
+ assert_greeting_with_assistant_message_response(response.messages)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
+ assert_greeting_with_assistant_message_response(messages_from_db, from_db=True)
+
+
+def test_greeting_without_assistant_message(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ llm_config: LLMConfig,
+ agent_state: AgentState,
+ json_metadata: pytest.FixtureRequest,
+) -> None:
+ """
+ Tests sending a message with a synchronous client.
+ Verifies that the response messages follow the expected order.
+ """
+ json_metadata["llm_config"] = dict(llm_config)
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_FORCE_REPLY,
+ use_assistant_message=False,
+ )
+ assert_greeting_without_assistant_message_response(response.messages)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id, use_assistant_message=False)
+ assert_greeting_without_assistant_message_response(messages_from_db, from_db=True)
+
+
+def test_tool_call(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ llm_config: LLMConfig,
+ agent_state: AgentState,
+ json_metadata: pytest.FixtureRequest,
+) -> None:
+ """
+ Tests sending a message with a synchronous client.
+ Verifies that the response messages follow the expected order.
+ """
+ json_metadata["llm_config"] = dict(llm_config)
+ dice_tool = client.tools.upsert_from_function(func=roll_dice)
+ client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_ROLL_DICE,
+ )
+ assert_tool_call_response(response.messages)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
+ assert_tool_call_response(messages_from_db, from_db=True)
+
+
+def test_url_image_input(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ llm_config: LLMConfig,
+ agent_state: AgentState,
+ json_metadata: pytest.FixtureRequest,
+) -> None:
+ """
+ Tests sending a message with a synchronous client.
+ Verifies that the response messages follow the expected order.
+ """
+ json_metadata["llm_config"] = dict(llm_config)
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_URL_IMAGE,
+ )
+ assert_image_input_response(response.messages)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
+ assert_image_input_response(messages_from_db, from_db=True)
+
+
+def test_base64_image_input(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ llm_config: LLMConfig,
+ agent_state: AgentState,
+ json_metadata: pytest.FixtureRequest,
+) -> None:
+ """
+ Tests sending a message with a synchronous client.
+ Verifies that the response messages follow the expected order.
+ """
+ json_metadata["llm_config"] = dict(llm_config)
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_BASE64_IMAGE,
+ )
+ assert_image_input_response(response.messages)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
+ assert_image_input_response(messages_from_db, from_db=True)
+
+
+def test_agent_loop_error(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ llm_config: LLMConfig,
+ agent_state: AgentState,
+ json_metadata: pytest.FixtureRequest,
+) -> None:
+ """
+ Tests sending a message with a synchronous client.
+ Verifies that no new messages are persisted on error.
+ """
+ json_metadata["llm_config"] = dict(llm_config)
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
+ tools = agent_state.tools
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config, tool_ids=[])
+ with pytest.raises(ApiError):
+ client.agents.messages.create(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_FORCE_REPLY,
+ )
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
+ assert len(messages_from_db) == 0
+ client.agents.modify(agent_id=agent_state.id, tool_ids=[t.id for t in tools])
+
+
+def test_step_streaming_greeting_with_assistant_message(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ llm_config: LLMConfig,
+ agent_state: AgentState,
+ json_metadata: pytest.FixtureRequest,
+) -> None:
+ """
+ Tests sending a streaming message with a synchronous client.
+ Checks that each chunk in the stream has the correct message types.
+ """
+ json_metadata["llm_config"] = dict(llm_config)
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_FORCE_REPLY,
+ )
+ chunks = list(response)
+ messages = accumulate_chunks(chunks)
+ assert_greeting_with_assistant_message_response(messages, streaming=True)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
+ assert_greeting_with_assistant_message_response(messages_from_db, from_db=True)
+
+
+def test_step_streaming_greeting_without_assistant_message(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ llm_config: LLMConfig,
+ agent_state: AgentState,
+ json_metadata: pytest.FixtureRequest,
+) -> None:
+ """
+ Tests sending a streaming message with a synchronous client.
+ Checks that each chunk in the stream has the correct message types.
+ """
+ json_metadata["llm_config"] = dict(llm_config)
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_FORCE_REPLY,
+ use_assistant_message=False,
+ )
+ chunks = list(response)
+ messages = accumulate_chunks(chunks)
+ assert_greeting_without_assistant_message_response(messages, streaming=True)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id, use_assistant_message=False)
+ assert_greeting_without_assistant_message_response(messages_from_db, from_db=True)
+
+
+def test_step_streaming_tool_call(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ llm_config: LLMConfig,
+ agent_state: AgentState,
+ json_metadata: pytest.FixtureRequest,
+) -> None:
+ """
+ Tests sending a streaming message with a synchronous client.
+ Checks that each chunk in the stream has the correct message types.
+ """
+ json_metadata["llm_config"] = dict(llm_config)
+ dice_tool = client.tools.upsert_from_function(func=roll_dice)
+ agent_state = client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_ROLL_DICE,
+ )
+ chunks = list(response)
+ messages = accumulate_chunks(chunks)
+ assert_tool_call_response(messages, streaming=True)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
+ assert_tool_call_response(messages_from_db, from_db=True)
+
+
+def test_step_stream_agent_loop_error(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ llm_config: LLMConfig,
+ agent_state: AgentState,
+ json_metadata: pytest.FixtureRequest,
+) -> None:
+ """
+ Tests sending a message with a synchronous client.
+ Verifies that no new messages are persisted on error.
+ """
+ json_metadata["llm_config"] = dict(llm_config)
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
+ tools = agent_state.tools
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config, tool_ids=[])
+ with pytest.raises(ApiError):
+ response = client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_FORCE_REPLY,
+ )
+ list(response)
+
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
+ assert len(messages_from_db) == 0
+ client.agents.modify(agent_id=agent_state.id, tool_ids=[t.id for t in tools])
+
+
+def test_token_streaming_greeting_with_assistant_message(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ llm_config: LLMConfig,
+ agent_state: AgentState,
+ json_metadata: pytest.FixtureRequest,
+) -> None:
+ """
+ Tests sending a streaming message with a synchronous client.
+ Checks that each chunk in the stream has the correct message types.
+ """
+ json_metadata["llm_config"] = dict(llm_config)
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_FORCE_REPLY,
+ stream_tokens=True,
+ )
+ chunks = list(response)
+ messages = accumulate_chunks(chunks)
+ assert_greeting_with_assistant_message_response(messages, streaming=True, token_streaming=True)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
+ assert_greeting_with_assistant_message_response(messages_from_db, from_db=True)
+
+
+def test_token_streaming_greeting_without_assistant_message(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ llm_config: LLMConfig,
+ agent_state: AgentState,
+ json_metadata: pytest.FixtureRequest,
+) -> None:
+ """
+ Tests sending a streaming message with a synchronous client.
+ Checks that each chunk in the stream has the correct message types.
+ """
+ json_metadata["llm_config"] = dict(llm_config)
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_FORCE_REPLY,
+ use_assistant_message=False,
+ stream_tokens=True,
+ )
+ chunks = list(response)
+ messages = accumulate_chunks(chunks)
+ assert_greeting_without_assistant_message_response(messages, streaming=True, token_streaming=True)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id, use_assistant_message=False)
+ assert_greeting_without_assistant_message_response(messages_from_db, from_db=True)
+
+
+def test_token_streaming_tool_call(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ llm_config: LLMConfig,
+ agent_state: AgentState,
+ json_metadata: pytest.FixtureRequest,
+) -> None:
+ """
+ Tests sending a streaming message with a synchronous client.
+ Checks that each chunk in the stream has the correct message types.
+ """
+ json_metadata["llm_config"] = dict(llm_config)
+ dice_tool = client.tools.upsert_from_function(func=roll_dice)
+ agent_state = client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_ROLL_DICE,
+ stream_tokens=True,
+ )
+ chunks = list(response)
+ messages = accumulate_chunks(chunks)
+ assert_tool_call_response(messages, streaming=True)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
+ assert_tool_call_response(messages_from_db, from_db=True)
+
+
+def test_token_streaming_agent_loop_error(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ llm_config: LLMConfig,
+ agent_state: AgentState,
+ json_metadata: pytest.FixtureRequest,
+) -> None:
+ """
+ Tests sending a message with a synchronous client.
+ Verifies that no new messages are persisted on error.
+ """
+ json_metadata["llm_config"] = dict(llm_config)
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
+ tools = agent_state.tools
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config, tool_ids=[])
+ try:
+ response = client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_FORCE_REPLY,
+ stream_tokens=True,
+ )
+ list(response)
+ except:
+ pass # only some models throw an error TODO: make this consistent
+
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
+ assert len(messages_from_db) == 0
+ client.agents.modify(agent_id=agent_state.id, tool_ids=[t.id for t in tools])
+
+
+def test_async_greeting_with_assistant_message(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ llm_config: LLMConfig,
+ agent_state: AgentState,
+ json_metadata: pytest.FixtureRequest,
+) -> None:
+ """
+ Tests sending a message as an asynchronous job using the synchronous client.
+ Waits for job completion and asserts that the result messages are as expected.
+ """
+ json_metadata["llm_config"] = dict(llm_config)
+ client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+
+ run = client.agents.messages.create_async(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_FORCE_REPLY,
+ )
+ run = wait_for_run_completion(client, run.id)
+
+ result = run.metadata.get("result")
+ assert result is not None, "Run metadata missing 'result' key"
+
+ messages = result["messages"]
+ assert_tool_response_dict_messages(messages)
+
+
+def test_auto_summarize(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ llm_config: LLMConfig,
+ json_metadata: pytest.FixtureRequest,
+) -> None:
+ """Test that summarization is automatically triggered."""
+ json_metadata["llm_config"] = dict(llm_config)
+
+ # pydantic prevents us for overriding the context window paramter in the passed LLMConfig
+ new_llm_config = llm_config.model_dump()
+ new_llm_config["context_window"] = 3000
+ pinned_context_window_llm_config = LLMConfig(**new_llm_config)
+
+ send_message_tool = client.tools.list(name="send_message")[0]
+ temp_agent_state = client.agents.create(
+ include_base_tools=False,
+ tool_ids=[send_message_tool.id],
+ llm_config=pinned_context_window_llm_config,
+ embedding="letta/letta-free",
+ tags=["supervisor"],
+ )
+
+ philosophical_question = """
+You know, sometimes I wonder if the entire structure of our lives is built on a series of unexamined assumptions we just silently agreed to somewhere along the way—like how we all just decided that five days a week of work and two days of “rest” constitutes balance, or how 9-to-5 became the default rhythm of a meaningful life, or even how the idea of “success” got boiled down to job titles and property ownership and productivity metrics on a LinkedIn profile, when maybe none of that is actually what makes a life feel full, or grounded, or real. And then there’s the weird paradox of ambition, how we're taught to chase it like a finish line that keeps moving, constantly redefining itself right as you’re about to grasp it—because even when you get the job, or the degree, or the validation, there's always something next, something more, like a treadmill with invisible settings you didn’t realize were turned up all the way.
+
+And have you noticed how we rarely stop to ask who set those definitions for us? Like was there ever a council that decided, yes, owning a home by thirty-five and retiring by sixty-five is the universal template for fulfillment? Or did it just accumulate like cultural sediment over generations, layered into us so deeply that questioning it feels uncomfortable, even dangerous? And isn’t it strange that we spend so much of our lives trying to optimize things—our workflows, our diets, our sleep, our morning routines—as though the point of life is to operate more efficiently rather than to experience it more richly? We build these intricate systems, these rulebooks for being a “high-functioning” human, but where in all of that is the space for feeling lost, for being soft, for wandering without a purpose just because it’s a sunny day and your heart is tugging you toward nowhere in particular?
+
+Sometimes I lie awake at night and wonder if all the noise we wrap around ourselves—notifications, updates, performance reviews, even our internal monologues—might be crowding out the questions we were meant to live into slowly, like how to love better, or how to forgive ourselves, or what the hell we’re even doing here in the first place. And when you strip it all down—no goals, no KPIs, no curated identity—what’s actually left of us? Are we just a sum of the roles we perform, or is there something quieter underneath that we've forgotten how to hear?
+
+And if there is something underneath all of it—something real, something worth listening to—then how do we begin to uncover it, gently, without rushing or reducing it to another task on our to-do list?
+ """
+
+ MAX_ATTEMPTS = 10
+ prev_length = None
+
+ for attempt in range(MAX_ATTEMPTS):
+ client.agents.messages.create(
+ agent_id=temp_agent_state.id,
+ messages=[MessageCreate(role="user", content=philosophical_question)],
+ )
+
+ temp_agent_state = client.agents.retrieve(agent_id=temp_agent_state.id)
+ message_ids = temp_agent_state.message_ids
+ current_length = len(message_ids)
+
+ print("LENGTH OF IN_CONTEXT_MESSAGES:", current_length)
+
+ if prev_length is not None and current_length <= prev_length:
+ # TODO: Add more stringent checks here
+ print(f"Summarization was triggered, detected current_length {current_length} is at least prev_length {prev_length}.")
+ break
+
+ prev_length = current_length
+ else:
+ raise AssertionError("Summarization was not triggered after 10 messages")
diff --git a/.github/scripts/model-sweep/supported-models.mdx b/.github/scripts/model-sweep/supported-models.mdx
new file mode 100644
index 000000000..a8a203c70
--- /dev/null
+++ b/.github/scripts/model-sweep/supported-models.mdx
@@ -0,0 +1,4551 @@
+---
+title: Support Models
+generated: 2025-06-20T16:40:44.072054
+---
+
+# Supported Models
+
+## Overview
+
+Letta routinely runs automated scans against available providers and models. These are the results of the latest scan.
+
+Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
+
+
+## anthropic
+
+| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |
+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|
+| `claude-3-5-haiku-20241022` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_5_haiku_20241022_details) |
+| `claude-3-5-sonnet-20241022` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_5_sonnet_20241022_details) |
+| `claude-3-7-sonnet-20250219` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_7_sonnet_20250219_details) |
+| `claude-sonnet-4-20250514` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_sonnet_4_20250514_details) |
+| `claude-opus-4-20250514` | ✅ | ✅ | ⚠️ | 200,000 | 2025-06-20 | [View](#anthropic_claude_opus_4_20250514_details) |
+| `claude-3-5-sonnet-20240620` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_5_sonnet_20240620_details) |
+| `claude-3-haiku-20240307` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_haiku_20240307_details) |
+| `claude-3-opus-20240229` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_opus_20240229_details) |
+| `claude-3-sonnet-20240229` | ❌ | ❌ | ❌ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_sonnet_20240229_details) |
+
+
+## openai
+
+| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |
+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|
+| `gpt-4.1` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_details) |
+| `gpt-4.1-2025-04-14` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_2025_04_14_details) |
+| `gpt-4.1-nano-2025-04-14` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_nano_2025_04_14_details) |
+| `gpt-4o` | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_details) |
+| `gpt-4o-2024-05-13` | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_2024_05_13_details) |
+| `gpt-4-turbo` | ✅ | ✅ | ⚠️ | 8,192 | 2025-06-20 | [View](#openai_gpt_4_turbo_details) |
+| `gpt-4.1-mini` | ✅ | ✅ | ⚠️ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_mini_details) |
+| `gpt-4.5-preview` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | [View](#openai_gpt_4.5_preview_details) |
+| `gpt-4.5-preview-2025-02-27` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | [View](#openai_gpt_4.5_preview_2025_02_27_details) |
+| `gpt-4o-2024-08-06` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_2024_08_06_details) |
+| `gpt-4-0613` | ✅ | ✅ | ❌ | 8,192 | 2025-06-20 | [View](#openai_gpt_4_0613_details) |
+| `gpt-4-1106-preview` | ✅ | ✅ | ❌ | 128,000 | 2025-06-20 | [View](#openai_gpt_4_1106_preview_details) |
+| `gpt-4-turbo-2024-04-09` | ✅ | ⚠️ | ✅ | 128,000 | 2025-06-20 | [View](#openai_gpt_4_turbo_2024_04_09_details) |
+| `gpt-4.1-mini-2025-04-14` | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_mini_2025_04_14_details) |
+| `gpt-4.1-nano` | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_nano_details) |
+| `gpt-4o-2024-11-20` | ⚠️ | ✅ | ✅ | 8,192 | 2025-06-20 | [View](#openai_gpt_4o_2024_11_20_details) |
+| `gpt-4-turbo-preview` | ✅ | ⚠️ | ❌ | 128,000 | 2025-06-20 | [View](#openai_gpt_4_turbo_preview_details) |
+| `gpt-4-0125-preview` | ⚠️ | ✅ | ❌ | 128,000 | 2025-06-20 | [View](#openai_gpt_4_0125_preview_details) |
+| `gpt-4o-mini` | ⚠️ | ⚠️ | ⚠️ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_mini_details) |
+| `gpt-4o-mini-2024-07-18` | ⚠️ | ⚠️ | ❌ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_mini_2024_07_18_details) |
+| `gpt-4` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_gpt_4_details) |
+| `o1` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o1_details) |
+| `o1-2024-12-17` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o1_2024_12_17_details) |
+| `o3` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_details) |
+| `o3-2025-04-16` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_2025_04_16_details) |
+| `o3-mini` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_mini_details) |
+| `o3-mini-2025-01-31` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_mini_2025_01_31_details) |
+| `o3-pro` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_pro_details) |
+| `o3-pro-2025-06-10` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_pro_2025_06_10_details) |
+
+
+## google_ai
+
+| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |
+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|
+| `gemini-1.5-pro` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_pro_details) |
+| `gemini-1.5-pro-002` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_pro_002_details) |
+| `gemini-1.5-pro-latest` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_pro_latest_details) |
+| `gemini-2.5-flash-preview-04-17-thinking` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_flash_preview_04_17_thinking_details) |
+| `gemini-2.5-pro-preview-03-25` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_preview_03_25_details) |
+| `gemini-2.5-pro-preview-05-06` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_preview_05_06_details) |
+| `gemini-2.5-flash-preview-05-20` | ✅ | ⚠️ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_flash_preview_05_20_details) |
+| `gemini-2.0-flash-thinking-exp` | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_thinking_exp_details) |
+| `gemini-2.0-flash-thinking-exp-1219` | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_thinking_exp_1219_details) |
+| `gemini-2.0-flash-thinking-exp-01-21` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_thinking_exp_01_21_details) |
+| `gemini-2.5-flash-preview-04-17` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_flash_preview_04_17_details) |
+| `gemini-2.5-pro-preview-06-05` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_preview_06_05_details) |
+| `gemini-1.0-pro-vision-latest` | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | [View](#google_ai_gemini_1.0_pro_vision_latest_details) |
+| `gemini-1.5-flash` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_details) |
+| `gemini-1.5-flash-002` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_002_details) |
+| `gemini-1.5-flash-8b` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_8b_details) |
+| `gemini-1.5-flash-8b-001` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_8b_001_details) |
+| `gemini-1.5-flash-8b-latest` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_8b_latest_details) |
+| `gemini-1.5-flash-latest` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_latest_details) |
+| `gemini-2.0-flash` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_details) |
+| `gemini-2.0-flash-001` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_001_details) |
+| `gemini-2.0-flash-exp` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_exp_details) |
+| `gemini-2.0-flash-exp-image-generation` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_exp_image_generation_details) |
+| `gemini-2.0-flash-lite` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_lite_details) |
+| `gemini-2.0-flash-lite-001` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_lite_001_details) |
+| `gemini-2.0-flash-lite-preview` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_lite_preview_details) |
+| `gemini-2.0-flash-lite-preview-02-05` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_lite_preview_02_05_details) |
+| `gemini-2.0-flash-preview-image-generation` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_preview_image_generation_details) |
+| `gemini-2.0-pro-exp` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_pro_exp_details) |
+| `gemini-2.0-pro-exp-02-05` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_pro_exp_02_05_details) |
+| `gemini-2.5-flash-preview-tts` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#google_ai_gemini_2.5_flash_preview_tts_details) |
+| `gemini-2.5-pro-exp-03-25` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_exp_03_25_details) |
+| `gemini-2.5-pro-preview-tts` | ❌ | ❌ | ❌ | 65,536 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_preview_tts_details) |
+| `gemini-exp-1206` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_exp_1206_details) |
+| `gemini-pro-vision` | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | [View](#google_ai_gemini_pro_vision_details) |
+
+
+## letta
+
+| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |
+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|
+| `letta-free` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#letta_letta_free_details) |
+
+
+## together
+
+| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |
+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|
+| `Qwen/Qwen2.5-72B-Instruct-Turbo` | ✅ | ✅ | ⚠️ | 131,072 | 2025-06-20 | [View](#together_qwen_qwen2.5_72b_instruct_turbo_details) |
+| `arcee-ai/virtuoso-large` | ⚠️ | ✅ | ✅ | 131,072 | 2025-06-20 | [View](#together_arcee_ai_virtuoso_large_details) |
+| `Qwen/QwQ-32B` | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | [View](#together_qwen_qwq_32b_details) |
+| `Qwen/Qwen2.5-7B-Instruct-Turbo` | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | [View](#together_qwen_qwen2.5_7b_instruct_turbo_details) |
+| `Qwen/Qwen2.5-Coder-32B-Instruct` | ⚠️ | ✅ | ⚠️ | 16,384 | 2025-06-20 | [View](#together_qwen_qwen2.5_coder_32b_instruct_details) |
+| `arcee-ai/coder-large` | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | [View](#together_arcee_ai_coder_large_details) |
+| `arcee_ai/arcee-spotlight` | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | [View](#together_arcee_ai_arcee_spotlight_details) |
+| `meta-llama/Llama-3.2-3B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.2_3b_instruct_turbo_details) |
+| `meta-llama/Llama-3.3-70B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.3_70b_instruct_turbo_details) |
+| `meta-llama/Llama-3.3-70B-Instruct-Turbo-Free` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.3_70b_instruct_turbo_free_details) |
+| `meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 130,815 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details) |
+| `meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details) |
+| `nvidia/Llama-3.1-Nemotron-70B-Instruct-HF` | ⚠️ | ✅ | ❌ | 32,768 | 2025-06-20 | [View](#together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details) |
+| `arcee-ai/virtuoso-medium-v2` | ⚠️ | ⚠️ | ✅ | 131,072 | 2025-06-20 | [View](#together_arcee_ai_virtuoso_medium_v2_details) |
+| `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8` | ⚠️ | ❌ | ✅ | 1,048,576 | 2025-06-20 | [View](#together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details) |
+| `Qwen/Qwen3-235B-A22B-fp8-tput` | ⚠️ | ⚠️ | ❌ | 40,960 | 2025-06-20 | [View](#together_qwen_qwen3_235b_a22b_fp8_tput_details) |
+| `deepseek-ai/DeepSeek-V3` | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_v3_details) |
+| `meta-llama/Llama-4-Scout-17B-16E-Instruct` | ⚠️ | ⚠️ | ❌ | 1,048,576 | 2025-06-20 | [View](#together_meta_llama_llama_4_scout_17b_16e_instruct_details) |
+| `meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo` | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details) |
+| `mistralai/Mixtral-8x7B-Instruct-v0.1` | ⚠️ | ⚠️ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mixtral_8x7b_instruct_v0.1_details) |
+| `arcee-ai/caller` | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | [View](#together_arcee_ai_caller_details) |
+| `mistralai/Mistral-Small-24B-Instruct-2501` | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mistral_small_24b_instruct_2501_details) |
+| `NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details) |
+| `Qwen/Qwen2-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_qwen_qwen2_72b_instruct_details) |
+| `Qwen/Qwen2-VL-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_qwen_qwen2_vl_72b_instruct_details) |
+| `Qwen/Qwen2.5-VL-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_qwen_qwen2.5_vl_72b_instruct_details) |
+| `arcee-ai/arcee-blitz` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_arcee_ai_arcee_blitz_details) |
+| `arcee-ai/maestro-reasoning` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_arcee_ai_maestro_reasoning_details) |
+| `deepseek-ai/DeepSeek-R1` | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_details) |
+| `deepseek-ai/DeepSeek-R1-Distill-Llama-70B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_distill_llama_70b_details) |
+| `deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details) |
+| `deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details) |
+| `deepseek-ai/DeepSeek-R1-Distill-Qwen-14B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_distill_qwen_14b_details) |
+| `deepseek-ai/DeepSeek-V3-p-dp` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_v3_p_dp_details) |
+| `google/gemma-2-27b-it` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_google_gemma_2_27b_it_details) |
+| `lgai/exaone-3-5-32b-instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_lgai_exaone_3_5_32b_instruct_details) |
+| `lgai/exaone-deep-32b` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_lgai_exaone_deep_32b_details) |
+| `marin-community/marin-8b-instruct` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_marin_community_marin_8b_instruct_details) |
+| `meta-llama/Llama-3-70b-chat-hf` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_meta_llama_llama_3_70b_chat_hf_details) |
+| `meta-llama/Llama-3-8b-chat-hf` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_meta_llama_llama_3_8b_chat_hf_details) |
+| `meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details) |
+| `meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details) |
+| `meta-llama/Llama-Vision-Free` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_vision_free_details) |
+| `meta-llama/Meta-Llama-3-70B-Instruct-Turbo` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3_70b_instruct_turbo_details) |
+| `meta-llama/Meta-Llama-3-8B-Instruct-Lite` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3_8b_instruct_lite_details) |
+| `mistralai/Mistral-7B-Instruct-v0.1` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mistral_7b_instruct_v0.1_details) |
+| `mistralai/Mistral-7B-Instruct-v0.2` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mistral_7b_instruct_v0.2_details) |
+| `mistralai/Mistral-7B-Instruct-v0.3` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mistral_7b_instruct_v0.3_details) |
+| `perplexity-ai/r1-1776` | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | [View](#together_perplexity_ai_r1_1776_details) |
+| `scb10x/scb10x-llama3-1-typhoon2-70b-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details) |
+| `scb10x/scb10x-typhoon-2-1-gemma3-12b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details) |
+| `togethercomputer/MoA-1` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_togethercomputer_moa_1_details) |
+| `togethercomputer/MoA-1-Turbo` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_togethercomputer_moa_1_turbo_details) |
+| `togethercomputer/Refuel-Llm-V2` | ❌ | ❌ | ❌ | 16,384 | 2025-06-20 | [View](#together_togethercomputer_refuel_llm_v2_details) |
+| `togethercomputer/Refuel-Llm-V2-Small` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_togethercomputer_refuel_llm_v2_small_details) |
+
+
+## deepseek
+
+| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |
+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|
+| `deepseek-chat` | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | [View](#deepseek_deepseek_chat_details) |
+| `deepseek-reasoner` | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | [View](#deepseek_deepseek_reasoner_details) |
+
+
+## groq
+
+| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |
+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|
+| `allam-2-7b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_allam_2_7b_details) |
+| `compound-beta` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_compound_beta_details) |
+| `compound-beta-mini` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_compound_beta_mini_details) |
+| `deepseek-r1-distill-llama-70b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_deepseek_r1_distill_llama_70b_details) |
+| `distil-whisper-large-v3-en` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_distil_whisper_large_v3_en_details) |
+| `gemma2-9b-it` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_gemma2_9b_it_details) |
+| `llama-3.1-8b-instant` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama_3.1_8b_instant_details) |
+| `llama-3.3-70b-versatile` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama_3.3_70b_versatile_details) |
+| `llama-guard-3-8b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama_guard_3_8b_details) |
+| `llama3-70b-8192` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama3_70b_8192_details) |
+| `llama3-8b-8192` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama3_8b_8192_details) |
+| `meta-llama/llama-4-maverick-17b-128e-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_4_maverick_17b_128e_instruct_details) |
+| `meta-llama/llama-4-scout-17b-16e-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_4_scout_17b_16e_instruct_details) |
+| `meta-llama/llama-guard-4-12b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_guard_4_12b_details) |
+| `meta-llama/llama-prompt-guard-2-22m` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_prompt_guard_2_22m_details) |
+| `meta-llama/llama-prompt-guard-2-86m` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_prompt_guard_2_86m_details) |
+| `mistral-saba-24b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_mistral_saba_24b_details) |
+| `playai-tts` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_playai_tts_details) |
+| `playai-tts-arabic` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_playai_tts_arabic_details) |
+| `qwen-qwq-32b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_qwen_qwq_32b_details) |
+| `qwen/qwen3-32b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_qwen_qwen3_32b_details) |
+| `whisper-large-v3` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_whisper_large_v3_details) |
+| `whisper-large-v3-turbo` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_whisper_large_v3_turbo_details) |
+
+
+---
+
+# Detailed Test Results
+
+## anthropic
+
+<a id="anthropic_claude_3_5_haiku_20241022_details"></a>
+### claude-3-5-haiku-20241022
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="anthropic_claude_3_5_sonnet_20241022_details"></a>
+### claude-3-5-sonnet-20241022
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="anthropic_claude_3_7_sonnet_20250219_details"></a>
+### claude-3-7-sonnet-20250219
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="anthropic_claude_sonnet_4_20250514_details"></a>
+### claude-sonnet-4-20250514
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="anthropic_claude_opus_4_20250514_details"></a>
+### claude-opus-4-20250514
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="anthropic_claude_3_5_sonnet_20240620_details"></a>
+### claude-3-5-sonnet-20240620
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="anthropic_claude_3_haiku_20240307_details"></a>
+### claude-3-haiku-20240307
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="anthropic_claude_3_opus_20240229_details"></a>
+### claude-3-opus-20240229
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="anthropic_claude_3_sonnet_20240229_details"></a>
+### claude-3-sonnet-20240229
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+
+
+## openai
+
+<a id="openai_gpt_4.1_details"></a>
+### gpt-4.1
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4.1_2025_04_14_details"></a>
+### gpt-4.1-2025-04-14
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4.1_nano_2025_04_14_details"></a>
+### gpt-4.1-nano-2025-04-14
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4o_details"></a>
+### gpt-4o
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4o_2024_05_13_details"></a>
+### gpt-4o-2024-05-13
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4_turbo_details"></a>
+### gpt-4-turbo
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4.1_mini_details"></a>
+### gpt-4.1-mini
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4.5_preview_details"></a>
+### gpt-4.5-preview
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4.5_preview_2025_02_27_details"></a>
+### gpt-4.5-preview-2025-02-27
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4o_2024_08_06_details"></a>
+### gpt-4o-2024-08-06
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4_0613_details"></a>
+### gpt-4-0613
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4_1106_preview_details"></a>
+### gpt-4-1106-preview
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4_turbo_2024_04_09_details"></a>
+### gpt-4-turbo-2024-04-09
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4.1_mini_2025_04_14_details"></a>
+### gpt-4.1-mini-2025-04-14
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4.1_nano_details"></a>
+### gpt-4.1-nano
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4o_2024_11_20_details"></a>
+### gpt-4o-2024-11-20
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4_turbo_preview_details"></a>
+### gpt-4-turbo-preview
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4_0125_preview_details"></a>
+### gpt-4-0125-preview
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4o_mini_details"></a>
+### gpt-4o-mini
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4o_mini_2024_07_18_details"></a>
+### gpt-4o-mini-2024-07-18
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_gpt_4_details"></a>
+### gpt-4
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_o1_details"></a>
+### o1
+
+### Basic
+
+- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_o1_2024_12_17_details"></a>
+### o1-2024-12-17
+
+### Basic
+
+- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_o3_details"></a>
+### o3
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_o3_2025_04_16_details"></a>
+### o3-2025-04-16
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_o3_mini_details"></a>
+### o3-mini
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_o3_mini_2025_01_31_details"></a>
+### o3-mini-2025-01-31
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_o3_pro_details"></a>
+### o3-pro
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="openai_o3_pro_2025_06_10_details"></a>
+### o3-pro-2025-06-10
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+
+
+## google_ai
+
+<a id="google_ai_gemini_1.5_pro_details"></a>
+### gemini-1.5-pro
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_1.5_pro_002_details"></a>
+### gemini-1.5-pro-002
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_1.5_pro_latest_details"></a>
+### gemini-1.5-pro-latest
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.5_flash_preview_04_17_thinking_details"></a>
+### gemini-2.5-flash-preview-04-17-thinking
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.5_pro_preview_03_25_details"></a>
+### gemini-2.5-pro-preview-03-25
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.5_pro_preview_05_06_details"></a>
+### gemini-2.5-pro-preview-05-06
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.5_flash_preview_05_20_details"></a>
+### gemini-2.5-flash-preview-05-20
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.0_flash_thinking_exp_details"></a>
+### gemini-2.0-flash-thinking-exp
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.0_flash_thinking_exp_1219_details"></a>
+### gemini-2.0-flash-thinking-exp-1219
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.0_flash_thinking_exp_01_21_details"></a>
+### gemini-2.0-flash-thinking-exp-01-21
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.5_flash_preview_04_17_details"></a>
+### gemini-2.5-flash-preview-04-17
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.5_pro_preview_06_05_details"></a>
+### gemini-2.5-pro-preview-06-05
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_1.0_pro_vision_latest_details"></a>
+### gemini-1.0-pro-vision-latest
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_1.5_flash_details"></a>
+### gemini-1.5-flash
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_1.5_flash_002_details"></a>
+### gemini-1.5-flash-002
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_1.5_flash_8b_details"></a>
+### gemini-1.5-flash-8b
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_1.5_flash_8b_001_details"></a>
+### gemini-1.5-flash-8b-001
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_1.5_flash_8b_latest_details"></a>
+### gemini-1.5-flash-8b-latest
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_1.5_flash_latest_details"></a>
+### gemini-1.5-flash-latest
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.0_flash_details"></a>
+### gemini-2.0-flash
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.0_flash_001_details"></a>
+### gemini-2.0-flash-001
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.0_flash_exp_details"></a>
+### gemini-2.0-flash-exp
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.0_flash_exp_image_generation_details"></a>
+### gemini-2.0-flash-exp-image-generation
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.0_flash_lite_details"></a>
+### gemini-2.0-flash-lite
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.0_flash_lite_001_details"></a>
+### gemini-2.0-flash-lite-001
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.0_flash_lite_preview_details"></a>
+### gemini-2.0-flash-lite-preview
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.0_flash_lite_preview_02_05_details"></a>
+### gemini-2.0-flash-lite-preview-02-05
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.0_flash_preview_image_generation_details"></a>
+### gemini-2.0-flash-preview-image-generation
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.0_pro_exp_details"></a>
+### gemini-2.0-pro-exp
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.0_pro_exp_02_05_details"></a>
+### gemini-2.0-pro-exp-02-05
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.5_flash_preview_tts_details"></a>
+### gemini-2.5-flash-preview-tts
+
+### Basic
+
+- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.5_pro_exp_03_25_details"></a>
+### gemini-2.5-pro-exp-03-25
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_2.5_pro_preview_tts_details"></a>
+### gemini-2.5-pro-preview-tts
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_exp_1206_details"></a>
+### gemini-exp-1206
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="google_ai_gemini_pro_vision_details"></a>
+### gemini-pro-vision
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+
+
+## letta
+
+<a id="letta_letta_free_details"></a>
+### letta-free
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+
+
+## together
+
+<a id="together_qwen_qwen2.5_72b_instruct_turbo_details"></a>
+### Qwen/Qwen2.5-72B-Instruct-Turbo
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_arcee_ai_virtuoso_large_details"></a>
+### arcee-ai/virtuoso-large
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_qwen_qwq_32b_details"></a>
+### Qwen/QwQ-32B
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_qwen_qwen2.5_7b_instruct_turbo_details"></a>
+### Qwen/Qwen2.5-7B-Instruct-Turbo
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_qwen_qwen2.5_coder_32b_instruct_details"></a>
+### Qwen/Qwen2.5-Coder-32B-Instruct
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_arcee_ai_coder_large_details"></a>
+### arcee-ai/coder-large
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_arcee_ai_arcee_spotlight_details"></a>
+### arcee_ai/arcee-spotlight
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_meta_llama_llama_3.2_3b_instruct_turbo_details"></a>
+### meta-llama/Llama-3.2-3B-Instruct-Turbo
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_meta_llama_llama_3.3_70b_instruct_turbo_details"></a>
+### meta-llama/Llama-3.3-70B-Instruct-Turbo
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_meta_llama_llama_3.3_70b_instruct_turbo_free_details"></a>
+### meta-llama/Llama-3.3-70B-Instruct-Turbo-Free
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details"></a>
+### meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details"></a>
+### meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details"></a>
+### nvidia/Llama-3.1-Nemotron-70B-Instruct-HF
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_arcee_ai_virtuoso_medium_v2_details"></a>
+### arcee-ai/virtuoso-medium-v2
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details"></a>
+### meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_qwen_qwen3_235b_a22b_fp8_tput_details"></a>
+### Qwen/Qwen3-235B-A22B-fp8-tput
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_deepseek_ai_deepseek_v3_details"></a>
+### deepseek-ai/DeepSeek-V3
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_meta_llama_llama_4_scout_17b_16e_instruct_details"></a>
+### meta-llama/Llama-4-Scout-17B-16E-Instruct
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details"></a>
+### meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_mistralai_mixtral_8x7b_instruct_v0.1_details"></a>
+### mistralai/Mixtral-8x7B-Instruct-v0.1
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_arcee_ai_caller_details"></a>
+### arcee-ai/caller
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_mistralai_mistral_small_24b_instruct_2501_details"></a>
+### mistralai/Mistral-Small-24B-Instruct-2501
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details"></a>
+### NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_qwen_qwen2_72b_instruct_details"></a>
+### Qwen/Qwen2-72B-Instruct
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_qwen_qwen2_vl_72b_instruct_details"></a>
+### Qwen/Qwen2-VL-72B-Instruct
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_qwen_qwen2.5_vl_72b_instruct_details"></a>
+### Qwen/Qwen2.5-VL-72B-Instruct
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_arcee_ai_arcee_blitz_details"></a>
+### arcee-ai/arcee-blitz
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_arcee_ai_maestro_reasoning_details"></a>
+### arcee-ai/maestro-reasoning
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_deepseek_ai_deepseek_r1_details"></a>
+### deepseek-ai/DeepSeek-R1
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_deepseek_ai_deepseek_r1_distill_llama_70b_details"></a>
+### deepseek-ai/DeepSeek-R1-Distill-Llama-70B
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details"></a>
+### deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details"></a>
+### deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_deepseek_ai_deepseek_r1_distill_qwen_14b_details"></a>
+### deepseek-ai/DeepSeek-R1-Distill-Qwen-14B
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_deepseek_ai_deepseek_v3_p_dp_details"></a>
+### deepseek-ai/DeepSeek-V3-p-dp
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_google_gemma_2_27b_it_details"></a>
+### google/gemma-2-27b-it
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_lgai_exaone_3_5_32b_instruct_details"></a>
+### lgai/exaone-3-5-32b-instruct
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_lgai_exaone_deep_32b_details"></a>
+### lgai/exaone-deep-32b
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_marin_community_marin_8b_instruct_details"></a>
+### marin-community/marin-8b-instruct
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_meta_llama_llama_3_70b_chat_hf_details"></a>
+### meta-llama/Llama-3-70b-chat-hf
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_meta_llama_llama_3_8b_chat_hf_details"></a>
+### meta-llama/Llama-3-8b-chat-hf
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details"></a>
+### meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details"></a>
+### meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_meta_llama_llama_vision_free_details"></a>
+### meta-llama/Llama-Vision-Free
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_meta_llama_meta_llama_3_70b_instruct_turbo_details"></a>
+### meta-llama/Meta-Llama-3-70B-Instruct-Turbo
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_meta_llama_meta_llama_3_8b_instruct_lite_details"></a>
+### meta-llama/Meta-Llama-3-8B-Instruct-Lite
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_mistralai_mistral_7b_instruct_v0.1_details"></a>
+### mistralai/Mistral-7B-Instruct-v0.1
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_mistralai_mistral_7b_instruct_v0.2_details"></a>
+### mistralai/Mistral-7B-Instruct-v0.2
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_mistralai_mistral_7b_instruct_v0.3_details"></a>
+### mistralai/Mistral-7B-Instruct-v0.3
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_perplexity_ai_r1_1776_details"></a>
+### perplexity-ai/r1-1776
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details"></a>
+### scb10x/scb10x-llama3-1-typhoon2-70b-instruct
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details"></a>
+### scb10x/scb10x-typhoon-2-1-gemma3-12b
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_togethercomputer_moa_1_details"></a>
+### togethercomputer/MoA-1
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_togethercomputer_moa_1_turbo_details"></a>
+### togethercomputer/MoA-1-Turbo
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_togethercomputer_refuel_llm_v2_details"></a>
+### togethercomputer/Refuel-Llm-V2
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="together_togethercomputer_refuel_llm_v2_small_details"></a>
+### togethercomputer/Refuel-Llm-V2-Small
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+
+
+## deepseek
+
+<a id="deepseek_deepseek_chat_details"></a>
+### deepseek-chat
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="deepseek_deepseek_reasoner_details"></a>
+### deepseek-reasoner
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+
+
+## groq
+
+<a id="groq_allam_2_7b_details"></a>
+### allam-2-7b
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_compound_beta_details"></a>
+### compound-beta
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_compound_beta_mini_details"></a>
+### compound-beta-mini
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_deepseek_r1_distill_llama_70b_details"></a>
+### deepseek-r1-distill-llama-70b
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_distil_whisper_large_v3_en_details"></a>
+### distil-whisper-large-v3-en
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_gemma2_9b_it_details"></a>
+### gemma2-9b-it
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_llama_3.1_8b_instant_details"></a>
+### llama-3.1-8b-instant
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_llama_3.3_70b_versatile_details"></a>
+### llama-3.3-70b-versatile
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_llama_guard_3_8b_details"></a>
+### llama-guard-3-8b
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_llama3_70b_8192_details"></a>
+### llama3-70b-8192
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_llama3_8b_8192_details"></a>
+### llama3-8b-8192
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_meta_llama_llama_4_maverick_17b_128e_instruct_details"></a>
+### meta-llama/llama-4-maverick-17b-128e-instruct
+
+### Basic
+
+- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_meta_llama_llama_4_scout_17b_16e_instruct_details"></a>
+### meta-llama/llama-4-scout-17b-16e-instruct
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_meta_llama_llama_guard_4_12b_details"></a>
+### meta-llama/llama-guard-4-12b
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_meta_llama_llama_prompt_guard_2_22m_details"></a>
+### meta-llama/llama-prompt-guard-2-22m
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_meta_llama_llama_prompt_guard_2_86m_details"></a>
+### meta-llama/llama-prompt-guard-2-86m
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_mistral_saba_24b_details"></a>
+### mistral-saba-24b
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_playai_tts_details"></a>
+### playai-tts
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_playai_tts_arabic_details"></a>
+### playai-tts-arabic
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_qwen_qwq_32b_details"></a>
+### qwen-qwq-32b
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_qwen_qwen3_32b_details"></a>
+### qwen/qwen3-32b
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_whisper_large_v3_details"></a>
+### whisper-large-v3
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
+
+<a id="groq_whisper_large_v3_turbo_details"></a>
+### whisper-large-v3-turbo
+
+### Basic
+
+- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L463)
+- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L704)
+- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L731)
+- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L345)
+- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L368)
+- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L566)
+- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L488)
+- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L513)
+- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L539)
+- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L392)
+
+### Token Streaming
+
+- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L674)
+- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L593)
+- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L619)
+- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L646)
+
+### Multimodal
+
+- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L440)
+- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/main/.github/scripts/model-sweep/model_sweep.py#L417)
diff --git a/.github/workflows/model-sweep.yaml b/.github/workflows/model-sweep.yaml
new file mode 100644
index 000000000..e31d14268
--- /dev/null
+++ b/.github/workflows/model-sweep.yaml
@@ -0,0 +1,144 @@
+name: Model Sweep
+on:
+ workflow_dispatch:
+ inputs:
+ branch-name:
+ required: true
+ type: string
+
+jobs:
+ model-sweep:
+ runs-on: [self-hosted, medium]
+ services:
+ qdrant:
+ image: qdrant/qdrant
+ ports:
+ - 6333:6333
+ postgres:
+ image: pgvector/pgvector:pg17
+ ports:
+ - 5432:5432
+ env:
+ POSTGRES_HOST_AUTH_METHOD: trust
+ POSTGRES_DB: postgres
+ POSTGRES_USER: postgres
+ options: >-
+ --health-cmd pg_isready
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 5
+
+ steps:
+ - name: Check if gh is installed
+ run: |
+ if ! command -v gh >/dev/null 2>&1
+ then
+ echo "gh could not be found, installing now..."
+ # install gh cli
+ (type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) \
+ && sudo mkdir -p -m 755 /etc/apt/keyrings \
+ && out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \
+ && cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
+ && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
+ && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
+ && sudo apt update \
+ && sudo apt install gh -y
+ fi
+
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: Inject env vars into environment
+ run: |
+ # Get secrets and mask them before adding to environment
+ while IFS= read -r line || [[ -n "$line" ]]; do
+ if [[ -n "$line" ]]; then
+ value=$(echo "$line" | cut -d= -f2-)
+ echo "::add-mask::$value"
+ echo "$line" >> $GITHUB_ENV
+ fi
+ done < <(letta_secrets_helper --env dev --service ci)
+
+ - name: Install dependencies
+ shell: bash
+ run: poetry install --no-interaction --no-root ${{ inputs.install-args || '-E dev -E postgres -E external-tools -E tests -E cloud-tool-sandbox -E google' }}
+ - name: Migrate database
+ env:
+ LETTA_PG_PORT: 5432
+ LETTA_PG_USER: postgres
+ LETTA_PG_PASSWORD: postgres
+ LETTA_PG_DB: postgres
+ LETTA_PG_HOST: localhost
+ run: |
+ psql -h localhost -U postgres -d postgres -c 'CREATE EXTENSION vector'
+ poetry run alembic upgrade head
+
+ - name: Run integration tests
+ # if any of the 1000+ test cases fail, pytest reports exit code 1 and won't procces/upload the results
+ continue-on-error: true
+ env:
+ LETTA_PG_PORT: 5432
+ LETTA_PG_USER: postgres
+ LETTA_PG_PASSWORD: postgres
+ LETTA_PG_DB: postgres
+ LETTA_PG_HOST: localhost
+ LETTA_SERVER_PASS: test_server_token
+ OPENAI_API_KEY: ${{ env.OPENAI_API_KEY }}
+ ANTHROPIC_API_KEY: ${{ env.ANTHROPIC_API_KEY }}
+ AZURE_API_KEY: ${{ env.AZURE_API_KEY }}
+ AZURE_BASE_URL: ${{ secrets.AZURE_BASE_URL }}
+ GEMINI_API_KEY: ${{ env.GEMINI_API_KEY }}
+ COMPOSIO_API_KEY: ${{ env.COMPOSIO_API_KEY }}
+ GOOGLE_CLOUD_PROJECT: ${{ secrets.GOOGLE_CLOUD_PROJECT}}
+ GOOGLE_CLOUD_LOCATION: ${{ secrets.GOOGLE_CLOUD_LOCATION}}
+ DEEPSEEK_API_KEY: ${{ env.DEEPSEEK_API_KEY}}
+ LETTA_USE_EXPERIMENTAL: 1
+ run: |
+ poetry run pytest \
+ -s -vv \
+ .github/scripts/model-sweep/model_sweep.py \
+ --json-report --json-report-file=.github/scripts/model-sweep/model_sweep_report.json --json-report-indent=4
+
+ - name: Convert report to markdown
+ continue-on-error: true
+ # file path args to generate_model_sweep_markdown.py are relative to the script
+ run: |
+ poetry run python \
+ .github/scripts/model-sweep/generate_model_sweep_markdown.py \
+ .github/scripts/model-sweep/model_sweep_report.json \
+ .github/scripts/model-sweep/supported-models.mdx
+ echo "Model sweep report saved to .github/scripts/model-sweep/supported-models.mdx"
+
+ - id: date
+ run: echo "date=$(date +%Y-%m-%d)" >> $GITHUB_OUTPUT
+
+ - name: commit and open pull request
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ BRANCH_NAME=model-sweep/${{ inputs.branch-name || format('{0}', steps.date.outputs.date) }}
+ gh auth setup-git
+ git config --global user.name "github-actions[bot]"
+ git config --global user.email "github-actions[bot]@users.noreply.github.com"
+ git checkout -b $BRANCH_NAME
+ git add .github/scripts/model-sweep/supported-models.mdx
+ git commit -m "Update model sweep report"
+ # only push if changes were made
+ if git diff main --quiet; then
+ echo "No changes detected, skipping push"
+ exit 0
+ else
+ git push origin $BRANCH_NAME
+ gh pr create \
+ --base main \
+ --head $BRANCH_NAME \
+ --title "chore: update model sweep report" \
+ --body "Automated PR to update model sweep report"
+ fi
+
+ - name: Upload model sweep report
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: model-sweep-report
+ path: .github/scripts/model-sweep/model_sweep_report.json
diff --git a/.github/workflows/send-message-integration-tests.yaml b/.github/workflows/send-message-integration-tests.yaml
new file mode 100644
index 000000000..eee96265d
--- /dev/null
+++ b/.github/workflows/send-message-integration-tests.yaml
@@ -0,0 +1,155 @@
+name: Send Message SDK Tests
+on:
+ pull_request_target:
+ # branches: [main] # TODO: uncomment before merge
+ types: [labeled]
+ paths:
+ - 'letta/**'
+
+jobs:
+ send-messages:
+ # Only run when the "safe to test" label is applied
+ if: contains(github.event.pull_request.labels.*.name, 'safe to test')
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ strategy:
+ fail-fast: false
+ matrix:
+ config_file:
+ - "openai-gpt-4o-mini.json"
+ - "azure-gpt-4o-mini.json"
+ - "claude-3-5-sonnet.json"
+ - "claude-3-7-sonnet.json"
+ - "claude-3-7-sonnet-extended.json"
+ - "gemini-pro.json"
+ - "gemini-vertex.json"
+ services:
+ qdrant:
+ image: qdrant/qdrant
+ ports:
+ - 6333:6333
+ postgres:
+ image: pgvector/pgvector:pg17
+ ports:
+ - 5432:5432
+ env:
+ POSTGRES_HOST_AUTH_METHOD: trust
+ POSTGRES_DB: postgres
+ POSTGRES_USER: postgres
+ options: >-
+ --health-cmd pg_isready
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 5
+
+ steps:
+ # Ensure secrets don't leak
+ - name: Configure git to hide secrets
+ run: |
+ git config --global core.logAllRefUpdates false
+ git config --global log.hideCredentials true
+ - name: Set up secret masking
+ run: |
+ # Automatically mask any environment variable ending with _KEY
+ for var in $(env | grep '_KEY=' | cut -d= -f1); do
+ value="${!var}"
+ if [[ -n "$value" ]]; then
+ # Mask the full value
+ echo "::add-mask::$value"
+
+ # Also mask partial values (first and last several characters)
+ # This helps when only parts of keys appear in logs
+ if [[ ${#value} -gt 8 ]]; then
+ echo "::add-mask::${value:0:8}"
+ echo "::add-mask::${value:(-8)}"
+ fi
+
+ # Also mask with common formatting changes
+ # Some logs might add quotes or other characters
+ echo "::add-mask::\"$value\""
+ echo "::add-mask::$value\""
+ echo "::add-mask::\"$value"
+
+ echo "Masked secret: $var (length: ${#value})"
+ fi
+ done
+
+ # Check out base repository code, not the PR's code (for security)
+ - name: Checkout base repository
+ uses: actions/checkout@v4 # No ref specified means it uses base branch
+
+ # Only extract relevant files from the PR (for security, specifically prevent modification of workflow files)
+ - name: Extract PR schema files
+ run: |
+ # Fetch PR without checking it out
+ git fetch origin pull/${{ github.event.pull_request.number }}/head:pr-${{ github.event.pull_request.number }}
+
+ # Extract ONLY the schema files
+ git checkout pr-${{ github.event.pull_request.number }} -- letta/
+ - name: Set up python 3.12
+ id: setup-python
+ uses: actions/setup-python@v5
+ with:
+ python-version: 3.12
+ - name: Load cached Poetry Binary
+ id: cached-poetry-binary
+ uses: actions/cache@v4
+ with:
+ path: ~/.local
+ key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-1.8.3
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+ with:
+ version: 1.8.3
+ virtualenvs-create: true
+ virtualenvs-in-project: true
+ - name: Load cached venv
+ id: cached-poetry-dependencies
+ uses: actions/cache@v4
+ with:
+ path: .venv
+ key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}${{ inputs.install-args || '-E dev -E postgres -E external-tools -E tests -E cloud-tool-sandbox' }}
+ # Restore cache with this prefix if not exact match with key
+ # Note cache-hit returns false in this case, so the below step will run
+ restore-keys: |
+ venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-
+ - name: Install dependencies
+ if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+ shell: bash
+ run: poetry install --no-interaction --no-root ${{ inputs.install-args || '-E dev -E postgres -E external-tools -E tests -E cloud-tool-sandbox -E google' }}
+ - name: Install letta packages via Poetry
+ run: |
+ poetry run pip install --upgrade letta-client letta
+ - name: Migrate database
+ env:
+ LETTA_PG_PORT: 5432
+ LETTA_PG_USER: postgres
+ LETTA_PG_PASSWORD: postgres
+ LETTA_PG_DB: postgres
+ LETTA_PG_HOST: localhost
+ run: |
+ psql -h localhost -U postgres -d postgres -c 'CREATE EXTENSION vector'
+ poetry run alembic upgrade head
+ - name: Run integration tests for ${{ matrix.config_file }}
+ env:
+ LLM_CONFIG_FILE: ${{ matrix.config_file }}
+ LETTA_PG_PORT: 5432
+ LETTA_PG_USER: postgres
+ LETTA_PG_PASSWORD: postgres
+ LETTA_PG_DB: postgres
+ LETTA_PG_HOST: localhost
+ LETTA_SERVER_PASS: test_server_token
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
+ AZURE_BASE_URL: ${{ secrets.AZURE_BASE_URL }}
+ GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+ COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }}
+ DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
+ GOOGLE_CLOUD_PROJECT: ${{ secrets.GOOGLE_CLOUD_PROJECT }}
+ GOOGLE_CLOUD_LOCATION: ${{ secrets.GOOGLE_CLOUD_LOCATION }}
+ run: |
+ poetry run pytest \
+ -s -vv \
+ tests/integration_test_send_message.py \
+ --maxfail=1 --durations=10
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 760be895f..3ea08c3c6 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -28,7 +28,7 @@ First, install Poetry using [the official instructions here](https://python-poet
Once Poetry is installed, navigate to the letta directory and install the Letta project with Poetry:
```shell
cd letta
-poetry shell
+eval $(poetry env activate)
poetry install --all-extras
```
#### Setup PostgreSQL environment (optional)
diff --git a/README.md b/README.md
index 042d476e8..1698d18c7 100644
--- a/README.md
+++ b/README.md
@@ -8,26 +8,13 @@
<div align="center">
<h1>Letta (previously MemGPT)</h1>
-
-**☄️ New release: Letta Agent Development Environment (_read more [here](#-access-the-ade-agent-development-environment)_) ☄️**
-
-<p align="center">
- <picture>
- <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/letta-ai/letta/refs/heads/main/assets/example_ade_screenshot.png">
- <source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/letta-ai/letta/refs/heads/main/assets/example_ade_screenshot_light.png">
- <img alt="Letta logo" src="https://raw.githubusercontent.com/letta-ai/letta/refs/heads/main/assets/example_ade_screenshot.png" width="800">
- </picture>
-</p>
-
----
-
<h3>
[Homepage](https://letta.com) // [Documentation](https://docs.letta.com) // [ADE](https://docs.letta.com/agent-development-environment) // [Letta Cloud](https://forms.letta.com/early-access)
</h3>
-**👾 Letta** is an open source framework for building stateful LLM applications. You can use Letta to build **stateful agents** with advanced reasoning capabilities and transparent long-term memory. The Letta framework is white box and model-agnostic.
+**👾 Letta** is an open source framework for building **stateful agents** with advanced reasoning capabilities and transparent long-term memory. The Letta framework is white box and model-agnostic.
[![Discord](https://img.shields.io/discord/1161736243340640419?label=Discord&logo=discord&logoColor=5865F2&style=flat-square&color=5865F2)](https://discord.gg/letta)
[![Twitter Follow](https://img.shields.io/badge/Follow-%40Letta__AI-1DA1F2?style=flat-square&logo=x&logoColor=white)](https://twitter.com/Letta_AI)
@@ -157,7 +144,7 @@ No, the data in your Letta server database stays on your machine. The Letta ADE
> _"Do I have to use your ADE? Can I build my own?"_
-The ADE is built on top of the (fully open source) Letta server and Letta Agents API. You can build your own application like the ADE on top of the REST API (view the documention [here](https://docs.letta.com/api-reference)).
+The ADE is built on top of the (fully open source) Letta server and Letta Agents API. You can build your own application like the ADE on top of the REST API (view the documentation [here](https://docs.letta.com/api-reference)).
> _"Can I interact with Letta agents via the CLI?"_
diff --git a/dev-compose.yaml b/dev-compose.yaml
index 42239b987..36fd5c549 100644
--- a/dev-compose.yaml
+++ b/dev-compose.yaml
@@ -28,7 +28,6 @@ services:
- "8083:8083"
- "8283:8283"
environment:
- - SERPAPI_API_KEY=${SERPAPI_API_KEY}
- LETTA_PG_DB=${LETTA_PG_DB:-letta}
- LETTA_PG_USER=${LETTA_PG_USER:-letta}
- LETTA_PG_PASSWORD=${LETTA_PG_PASSWORD:-letta}
diff --git a/examples/docs/example.py b/examples/docs/example.py
index 2f3530a28..990d846d6 100644
--- a/examples/docs/example.py
+++ b/examples/docs/example.py
@@ -8,6 +8,8 @@ If you're using Letta Cloud, replace 'baseURL' with 'token'
See: https://docs.letta.com/api-reference/overview
Execute this script using `poetry run python3 example.py`
+
+This will install `letta_client` and other dependencies.
"""
client = Letta(
base_url="http://localhost:8283",
diff --git a/examples/files/README.md b/examples/files/README.md
new file mode 100644
index 000000000..e6b4a421c
--- /dev/null
+++ b/examples/files/README.md
@@ -0,0 +1,34 @@
+# Letta Files and Streaming Demo
+
+This demo shows how to work with Letta's file upload and streaming capabilities.
+
+## Features
+
+- Upload files from disk to a Letta data source
+- Create files from strings and upload them
+- Download and upload PDF files
+- Create an agent and attach data sources
+- Stream agent responses in real-time
+- Interactive chat with file-aware agent
+
+## Files
+
+- `main.py` - Main demo script showing file upload and streaming
+- `example-on-disk.txt` - Sample text file for upload demonstration
+- `memgpt.pdf` - MemGPT paper (downloaded automatically)
+
+## Setup
+
+1. Set your Letta API key: `export LETTA_API_KEY=your_key_here`
+2. Install dependencies: `pip install letta-client requests rich`
+3. Run the demo: `python main.py`
+
+## Usage
+
+The demo will:
+1. Create a data source called "Example Source"
+2. Upload the example text file and PDF
+3. Create an agent named "Clippy"
+4. Start an interactive chat session
+
+Type 'quit' or 'exit' to end the conversation.
diff --git a/examples/files/example-on-disk.txt b/examples/files/example-on-disk.txt
new file mode 100644
index 000000000..d8f9b2b7a
--- /dev/null
+++ b/examples/files/example-on-disk.txt
@@ -0,0 +1,2 @@
+Hey, you're looking at a different example.
+This password is "stateful agents".
diff --git a/examples/files/main.py b/examples/files/main.py
new file mode 100644
index 000000000..6ef049784
--- /dev/null
+++ b/examples/files/main.py
@@ -0,0 +1,190 @@
+"""
+Letta Filesystem
+
+This demo shows how to:
+1. Create a folder and upload files (both from disk and from strings)
+2. Create an agent and attach the data folder
+3. Stream the agent's responses
+4. Query the agent about the uploaded files
+
+The demo uploads:
+- A text file from disk (example-on-disk.txt)
+- A text file created from a string (containing a password)
+- The MemGPT paper PDF from arXiv
+
+Then asks the agent to summarize the paper and find passwords in the files.
+"""
+
+import os
+
+import requests
+from letta_client import Letta
+from letta_client.core.api_error import ApiError
+from rich import print
+
+LETTA_API_KEY = os.getenv("LETTA_API_KEY")
+if LETTA_API_KEY is None:
+ raise ValueError("LETTA_API_KEY is not set")
+
+FOLDER_NAME = "Example Folder"
+
+# Connect to our Letta server
+client = Letta(token=LETTA_API_KEY)
+
+# get an available embedding_config
+embedding_configs = client.embedding_models.list()
+embedding_config = embedding_configs[0]
+
+# Check if the folder already exists
+try:
+ folder_id = client.folders.retrieve_by_name(FOLDER_NAME)
+
+# We got an API error. Check if it's a 404, meaning the folder doesn't exist.
+except ApiError as e:
+ if e.status_code == 404:
+ # Create a new folder
+ folder = client.folders.create(
+ name=FOLDER_NAME,
+ description="This is an example folder",
+ instructions="Use this data folder to see how Letta works.",
+ )
+ folder_id = folder.id
+ else:
+ raise e
+
+except Exception as e:
+ # Something else went wrong
+ raise e
+
+
+#
+# There's two ways to upload a file to a folder.
+#
+# 1. From an existing file
+# 2. From a string by encoding it into a base64 string
+#
+#
+
+# 1. From an existing file
+# "rb" means "read binary"
+file = open("example-on-disk.txt", "rb")
+
+# Upload the file to the folder
+file = client.folders.files.upload(
+ folder_id=folder_id,
+ file=file,
+ duplicate_handling="skip"
+)
+
+# 2. From a string by encoding it into a base64 string
+import io
+
+content = """
+This is an example file. If you can read this,
+the password is 'letta'.
+"""
+
+# Encode the string into bytes, and then create a file-like object
+# that exists only in memory.
+file_object = io.BytesIO(content.encode("utf-8"))
+
+# Set the name of the file
+file_object.name = "example.txt"
+
+# Upload the file to the folder
+file = client.folders.files.upload(
+ folder_id=folder_id,
+ file=file_object,
+ duplicate_handling="skip"
+)
+
+#
+# You can also upload PDFs!
+# Letta extracts text from PDFs using OCR.
+#
+
+# Download the PDF to the local directory if it doesn't exist
+if not os.path.exists("memgpt.pdf"):
+ # Download the PDF
+ print("Downloading memgpt.pdf")
+ response = requests.get("https://arxiv.org/pdf/2310.08560")
+ with open("memgpt.pdf", "wb") as f:
+ f.write(response.content)
+
+# Upload the PDF to the folder
+file = client.folders.files.upload(
+ folder_id=folder_id,
+ file=open("memgpt.pdf", "rb"),
+ duplicate_handling="skip"
+)
+
+#
+# Now we need to create an agent that can use this folder
+#
+
+# Create an agent
+agent = client.agents.create(
+ model="openai/gpt-4o-mini",
+ name="Example Agent",
+ description="This agent looks at files and answers questions about them.",
+ memory_blocks = [
+ {
+ "label": "human",
+ "value": "The human wants to know about the files."
+ },
+ {
+ "label": "persona",
+ "value": "My name is Clippy, I answer questions about files."
+ }
+ ]
+)
+
+# Attach the data folder to the agent.
+# Once the folder is attached, the agent will be able to see all
+# files in the folder.
+client.agents.folders.attach(
+ agent_id=agent.id,
+ folder_id=folder_id
+)
+
+########################################################
+# This code makes a simple chatbot interface to the agent
+########################################################
+
+# Wrap this in a try/catch block to remove the agent in the event of an error
+try:
+ print(f"🤖 Connected to agent: {agent.name}")
+ print("💡 Type 'quit' or 'exit' to end the conversation")
+ print("=" * 50)
+
+ while True:
+ # Get user input
+ try:
+ user_input = input("\n👤 You: ").strip()
+ except (EOFError, KeyboardInterrupt):
+ print("\n👋 Goodbye!")
+ break
+
+ if user_input.lower() in ['quit', 'exit', 'q']:
+ print("👋 Goodbye!")
+ break
+
+ if not user_input:
+ continue
+
+ # Stream the agent's response
+ stream = client.agents.messages.create_stream(
+ agent_id=agent.id,
+ messages=[
+ {
+ "role": "user",
+ "content": user_input
+ }
+ ],
+ )
+
+ for chunk in stream:
+ print(chunk)
+
+finally:
+ client.agents.delete(agent.id)
diff --git a/examples/mcp_example.py b/examples/mcp_example.py
index 9352a1a37..25d1aaf81 100644
--- a/examples/mcp_example.py
+++ b/examples/mcp_example.py
@@ -2,22 +2,33 @@ from pprint import pprint
from letta_client import Letta
+# Connect to Letta server
client = Letta(base_url="http://localhost:8283")
+# Use the "everything" mcp server:
+# https://github.com/modelcontextprotocol/servers/tree/main/src/everything
mcp_server_name = "everything"
mcp_tool_name = "echo"
+# List all McpTool belonging to the "everything" mcp server.
mcp_tools = client.tools.list_mcp_tools_by_server(
mcp_server_name=mcp_server_name,
)
+
+# We can see that "echo" is one of the tools, but it's not
+# a letta tool that can be added to a client (it has no tool id).
for tool in mcp_tools:
pprint(tool)
+# Create a Tool (with a tool id) using the server and tool names.
mcp_tool = client.tools.add_mcp_tool(
mcp_server_name=mcp_server_name,
mcp_tool_name=mcp_tool_name
)
+# Create an agent with the tool, using tool.id -- note that
+# this is the ONLY tool in the agent, you typically want to
+# also include the default tools.
agent = client.agents.create(
memory_blocks=[
{
@@ -31,6 +42,7 @@ agent = client.agents.create(
)
print(f"Created agent id {agent.id}")
+# Ask the agent to call the tool.
response = client.agents.messages.create(
agent_id=agent.id,
messages=[
diff --git a/examples/notebooks/Customizing memory management.ipynb b/examples/notebooks/Customizing memory management.ipynb
index af167dce3..2df343b4d 100644
--- a/examples/notebooks/Customizing memory management.ipynb
+++ b/examples/notebooks/Customizing memory management.ipynb
@@ -253,15 +253,18 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": null,
"id": "7808912f-831b-4cdc-8606-40052eb809b4",
"metadata": {},
"outputs": [],
"source": [
- "from typing import Optional, List\n",
+ "from typing import Optional, List, TYPE_CHECKING\n",
"import json\n",
"\n",
- "def task_queue_push(self: \"Agent\", task_description: str):\n",
+ "if TYPE_CHECKING:\n",
+ " from letta import AgentState\n",
+ "\n",
+ "def task_queue_push(agent_state: \"AgentState\", task_description: str):\n",
" \"\"\"\n",
" Push to a task queue stored in core memory. \n",
"\n",
@@ -273,12 +276,12 @@
" does not produce a response.\n",
" \"\"\"\n",
" import json\n",
- " tasks = json.loads(self.memory.get_block(\"tasks\").value)\n",
+ " tasks = json.loads(agent_state.memory.get_block(\"tasks\").value)\n",
" tasks.append(task_description)\n",
- " self.memory.update_block_value(\"tasks\", json.dumps(tasks))\n",
+ " agent_state.memory.update_block_value(\"tasks\", json.dumps(tasks))\n",
" return None\n",
"\n",
- "def task_queue_pop(self: \"Agent\"):\n",
+ "def task_queue_pop(agent_state: \"AgentState\"):\n",
" \"\"\"\n",
" Get the next task from the task queue \n",
"\n",
@@ -288,12 +291,12 @@
" None (the task queue is empty)\n",
" \"\"\"\n",
" import json\n",
- " tasks = json.loads(self.memory.get_block(\"tasks\").value)\n",
+ " tasks = json.loads(agent_state.memory.get_block(\"tasks\").value)\n",
" if len(tasks) == 0: \n",
" return None\n",
" task = tasks[0]\n",
" print(\"CURRENT TASKS: \", tasks)\n",
- " self.memory.update_block_value(\"tasks\", json.dumps(tasks[1:]))\n",
+ " agent_state.memory.update_block_value(\"tasks\", json.dumps(tasks[1:]))\n",
" return task\n",
"\n",
"push_task_tool = client.tools.upsert_from_function(func=task_queue_push)\n",
@@ -310,7 +313,7 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": null,
"id": "135fcf3e-59c4-4da3-b86b-dbffb21aa343",
"metadata": {},
"outputs": [],
@@ -336,10 +339,12 @@
" ),\n",
" CreateBlock(\n",
" label=\"tasks\",\n",
- " value=\"\",\n",
+ " value=\"[]\",\n",
" ),\n",
" ],\n",
" tool_ids=[push_task_tool.id, pop_task_tool.id],\n",
+ " model=\"letta/letta-free\",\n",
+ " embedding=\"letta/letta-free\",\n",
")"
]
},
diff --git a/letta/__main__.py b/letta/__main__.py
deleted file mode 100644
index 89f11424b..000000000
--- a/letta/__main__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from .main import app
-
-app()
diff --git a/letta/functions/function_sets/base.py b/letta/functions/function_sets/base.py
index 233ce7a51..36ebec986 100644
--- a/letta/functions/function_sets/base.py
+++ b/letta/functions/function_sets/base.py
@@ -46,7 +46,7 @@ def conversation_search(self: "Agent", query: str, page: Optional[int] = 0) -> O
count = RETRIEVAL_QUERY_DEFAULT_PAGE_SIZE
# TODO: add paging by page number. currently cursor only works with strings.
# original: start=page * count
- messages = self.message_manager.list_user_messages_for_agent(
+ messages = self.message_manager.list_messages_for_agent(
agent_id=self.agent_state.id,
actor=self.user,
query_text=query,
diff --git a/letta/llm_api/anthropic_client.py b/letta/llm_api/anthropic_client.py
index 5a3206058..7b886dfca 100644
--- a/letta/llm_api/anthropic_client.py
+++ b/letta/llm_api/anthropic_client.py
@@ -242,7 +242,8 @@ class AnthropicClient(LLMClientBase):
# Move 'system' to the top level
if messages[0].role != "system":
raise RuntimeError(f"First message is not a system message, instead has role {messages[0].role}")
- data["system"] = messages[0].content if isinstance(messages[0].content, str) else messages[0].content[0].text
+ system_content = messages[0].content if isinstance(messages[0].content, str) else messages[0].content[0].text
+ data["system"] = self._add_cache_control_to_system_message(system_content)
data["messages"] = [
m.to_anthropic_dict(
inner_thoughts_xml_tag=inner_thoughts_xml_tag,
@@ -499,6 +500,22 @@ class AnthropicClient(LLMClientBase):
return chat_completion_response
+ def _add_cache_control_to_system_message(self, system_content):
+ """Add cache control to system message content"""
+ if isinstance(system_content, str):
+ # For string content, convert to list format with cache control
+ return [{"type": "text", "text": system_content, "cache_control": {"type": "ephemeral"}}]
+ elif isinstance(system_content, list):
+ # For list content, add cache control to the last text block
+ cached_content = system_content.copy()
+ for i in range(len(cached_content) - 1, -1, -1):
+ if cached_content[i].get("type") == "text":
+ cached_content[i]["cache_control"] = {"type": "ephemeral"}
+ break
+ return cached_content
+
+ return system_content
+
def convert_tools_to_anthropic_format(tools: List[OpenAITool]) -> List[dict]:
"""See: https://docs.anthropic.com/claude/docs/tool-use
diff --git a/letta/schemas/llm_config.py b/letta/schemas/llm_config.py
index 6d6b6039b..6259423fb 100644
--- a/letta/schemas/llm_config.py
+++ b/letta/schemas/llm_config.py
@@ -63,7 +63,8 @@ class LLMConfig(BaseModel):
description="The reasoning effort to use when generating text reasoning models",
)
max_reasoning_tokens: int = Field(
- 0, description="Configurable thinking budget for extended thinking, only used if enable_reasoner is True. Minimum value is 1024."
+ 0,
+ description="Configurable thinking budget for extended thinking. Used for enable_reasoner and also for Google Vertex models like Gemini 2.5 Flash. Minimum value is 1024 when used with enable_reasoner.",
)
frequency_penalty: Optional[float] = Field(
None, # Can also deafult to 0.0?
diff --git a/letta/schemas/providers/ollama.py b/letta/schemas/providers/ollama.py
index 8cc8f7202..df7d83468 100644
--- a/letta/schemas/providers/ollama.py
+++ b/letta/schemas/providers/ollama.py
@@ -1,7 +1,6 @@
from typing import Literal
import aiohttp
-import requests
from pydantic import Field
from letta.constants import DEFAULT_EMBEDDING_CHUNK_SIZE
@@ -43,7 +42,7 @@ class OllamaProvider(OpenAIProvider):
configs = []
for model in response_json["models"]:
- context_window = self.get_model_context_window(model["name"])
+ context_window = await self._get_model_context_window(model["name"])
if context_window is None:
print(f"Ollama model {model['name']} has no context window, using default 32000")
context_window = 32000
@@ -92,63 +91,50 @@ class OllamaProvider(OpenAIProvider):
)
return configs
- def get_model_context_window(self, model_name: str) -> int | None:
- """Gets model context window for Ollama. As this can look different based on models,
- we use the following for guidance:
-
- "llama.context_length": 8192,
- "llama.embedding_length": 4096,
- source: https://github.com/ollama/ollama/blob/main/docs/api.md#show-model-information
-
- FROM 2024-10-08
- Notes from vLLM around keys
- source: https://github.com/vllm-project/vllm/blob/72ad2735823e23b4e1cc79b7c73c3a5f3c093ab0/vllm/config.py#L3488
-
- possible_keys = [
- # OPT
- "max_position_embeddings",
- # GPT-2
- "n_positions",
- # MPT
- "max_seq_len",
- # ChatGLM2
- "seq_length",
- # Command-R
- "model_max_length",
- # Whisper
- "max_target_positions",
- # Others
- "max_sequence_length",
- "max_seq_length",
- "seq_len",
- ]
- max_position_embeddings
- parse model cards: nous, dolphon, llama
- """
+ async def _get_model_context_window(self, model_name: str) -> int | None:
endpoint = f"{self.base_url}/api/show"
- payload = {"name": model_name, "verbose": True}
- response = requests.post(endpoint, json=payload)
- if response.status_code != 200:
- return None
+ payload = {"name": model_name}
try:
- model_info = response.json()
- # Try to extract context window from model parameters
- if "model_info" in model_info and "llama.context_length" in model_info["model_info"]:
- return int(model_info["model_info"]["llama.context_length"])
- except Exception:
- pass
- logger.warning(f"Failed to get model context window for {model_name}")
+ async with aiohttp.ClientSession() as session:
+ async with session.post(endpoint, json=payload) as response:
+ if response.status != 200:
+ error_text = await response.text()
+ logger.warning(f"Failed to get model info for {model_name}: {response.status} - {error_text}")
+ return None
+
+ response_json = await response.json()
+ model_info = response_json.get("model_info", {})
+
+ if architecture := model_info.get("general.architecture"):
+ if context_length := model_info.get(f"{architecture}.context_length"):
+ return int(context_length)
+
+ except Exception as e:
+ logger.warning(f"Failed to get model context window for {model_name} with error: {e}")
+
return None
- async def _get_model_embedding_dim_async(self, model_name: str):
- async with aiohttp.ClientSession() as session:
- async with session.post(f"{self.base_url}/api/show", json={"name": model_name, "verbose": True}) as response:
- response_json = await response.json()
+ async def _get_model_embedding_dim(self, model_name: str) -> int | None:
+ endpoint = f"{self.base_url}/api/show"
+ payload = {"name": model_name}
+
+ try:
+ async with aiohttp.ClientSession() as session:
+ async with session.post(endpoint, json=payload) as response:
+ if response.status != 200:
+ error_text = await response.text()
+ logger.warning(f"Failed to get model info for {model_name}: {response.status} - {error_text}")
+ return None
- if "model_info" not in response_json:
- if "error" in response_json:
- logger.warning("Ollama fetch model info error for %s: %s", model_name, response_json["error"])
- return None
+ response_json = await response.json()
+ model_info = response_json.get("model_info", {})
- return response_json["model_info"].get("embedding_length")
+ if architecture := model_info.get("general.architecture"):
+ if embedding_length := model_info.get(f"{architecture}.embedding_length"):
+ return int(embedding_length)
+
+ except Exception as e:
+ logger.warning(f"Failed to get model embedding dimension for {model_name} with error: {e}")
+
+ return None
diff --git a/tests/test_base_functions.py b/tests/test_base_functions.py
index f267c61dc..7ba28ee90 100644
--- a/tests/test_base_functions.py
+++ b/tests/test_base_functions.py
@@ -151,6 +151,7 @@ def test_archival(agent_obj):
def test_recall(server, agent_obj, default_user):
"""Test that an agent can recall messages using a keyword via conversation search."""
keyword = "banana"
+ "".join(reversed(keyword))
# Send messages
for msg in ["hello", keyword, "tell me a fun fact"]: