Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

move results to bottom of the page

Kian Jones684913364d47
diff --git a/.github/scripts/model-sweep/generate_model_sweep_markdown.py b/.github/scripts/model-sweep/generate_model_sweep_markdown.py
index 09aafa7e5..9d965b576 100644
--- a/.github/scripts/model-sweep/generate_model_sweep_markdown.py
+++ b/.github/scripts/model-sweep/generate_model_sweep_markdown.py
@@ -355,8 +355,9 @@ def process_model_sweep_report(input_file, output_file, config_file=None, debug=
all_provider_data[provider_name] = model_data
provider_support_scores[provider_name] = calculate_provider_support_score(model_data, list(feature_mapping.keys()))
- # Calculate column widths for consistent formatting
+ # Calculate column widths for consistent formatting (add details column)
column_widths = calculate_column_widths(all_provider_data, feature_mapping)
+ column_widths['details'] = len('Details')
# Sort providers by support score (descending) then by name (ascending)
sorted_providers = sorted(
@@ -364,7 +365,7 @@ def process_model_sweep_report(input_file, output_file, config_file=None, debug=
key=lambda x: (-provider_support_scores[x], x)
)
- # Generate output for each provider
+ # Generate tables for all providers first
for provider_name in sorted_providers:
model_data = all_provider_data[provider_name]
support_score = provider_support_scores[provider_name]
@@ -378,7 +379,8 @@ def process_model_sweep_report(input_file, output_file, config_file=None, debug=
header_parts.append(f"{feature:^{column_widths[feature]}}")
header_parts.extend([
f"{'Context Window':^{column_widths['context_window']}}",
- f"{'Last Scanned':^{column_widths['last_scanned']}}"
+ f"{'Last Scanned':^{column_widths['last_scanned']}}",
+ f"{'Details':^{column_widths['details']}}"
])
header_row = "| " + " | ".join(header_parts) + " |"
@@ -388,7 +390,8 @@ def process_model_sweep_report(input_file, output_file, config_file=None, debug=
separator_parts.append(f":{'-' * (column_widths[feature] - 2)}:")
separator_parts.extend([
f":{'-' * (column_widths['context_window'] - 2)}:",
- f":{'-' * (column_widths['last_scanned'] - 2)}:"
+ f":{'-' * (column_widths['last_scanned'] - 2)}:",
+ f":{'-' * (column_widths['details'] - 2)}:"
])
separator_row = "|" + "|".join(separator_parts) + "|"
@@ -404,6 +407,7 @@ def process_model_sweep_report(input_file, output_file, config_file=None, debug=
for model_info in model_data:
# Create anchor for model details
model_anchor = model_info['name'].replace('/', '_').replace(':', '_').replace('-', '_').lower()
+ details_anchor = f"{provider_name.lower().replace(' ', '_')}_{model_anchor}_details"
# Build row with left-aligned first column, centered others
row_parts = [f"`{model_info['name']}`".ljust(column_widths['model'])]
@@ -411,25 +415,35 @@ def process_model_sweep_report(input_file, output_file, config_file=None, debug=
row_parts.append(f"{model_info['feature_support'][feature]:^{column_widths[feature]}}")
row_parts.extend([
f"{model_info['context_window']:,}".center(column_widths['context_window']),
- f"{model_info['last_scanned']}".center(column_widths['last_scanned'])
+ f"{model_info['last_scanned']}".center(column_widths['last_scanned']),
+ f"[View](#{details_anchor})".center(column_widths['details'])
])
row = "| " + " | ".join(row_parts) + " |"
mdx_lines.append(row)
- # Add detailed test results for each model
- mdx_lines.extend(["", "### Detailed Test Results", ""])
+ # Add spacing between provider tables
+ mdx_lines.extend(["", ""])
+
+ # Add detailed test results section after all tables
+ mdx_lines.extend(["---", "", "# Detailed Test Results", ""])
+
+ for provider_name in sorted_providers:
+ model_data = all_provider_data[provider_name]
+ mdx_lines.extend([f"## {provider_name}", ""])
for model_info in model_data:
model_anchor = model_info['name'].replace('/', '_').replace(':', '_').replace('-', '_').lower()
- mdx_lines.append(f"#### {model_info['name']}")
+ details_anchor = f"{provider_name.lower().replace(' ', '_')}_{model_anchor}_details"
+ mdx_lines.append(f"<a id=\"{details_anchor}\"></a>")
+ mdx_lines.append(f"### {model_info['name']}")
mdx_lines.append("")
# Add test details
test_details = generate_test_details(model_info, feature_mapping)
mdx_lines.extend(test_details)
- # Add spacing between providers
- mdx_lines.extend(["", "---", ""])
+ # Add spacing between providers in details section
+ mdx_lines.extend(["", ""])
# Write the MDX file
with open(output_file, 'w') as f:
diff --git a/.github/scripts/model-sweep/model_sweep_report.mdx b/.github/scripts/model-sweep/model_sweep_report.mdx
index 8990e4407..7d43ed2a4 100644
--- a/.github/scripts/model-sweep/model_sweep_report.mdx
+++ b/.github/scripts/model-sweep/model_sweep_report.mdx
@@ -1,6 +1,6 @@
---
title: Support Models
-generated: 2025-06-20T16:24:07.581430
+generated: 2025-06-20T16:32:55.429847
---
# Supported Models
@@ -14,21 +14,208 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
## anthropic
-| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned |
-|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|
-| `claude-3-5-haiku-20241022` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 |
-| `claude-3-5-sonnet-20241022` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 |
-| `claude-3-7-sonnet-20250219` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 |
-| `claude-sonnet-4-20250514` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 |
-| `claude-opus-4-20250514` | ✅ | ✅ | ⚠️ | 200,000 | 2025-06-20 |
-| `claude-3-5-sonnet-20240620` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 |
-| `claude-3-haiku-20240307` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 |
-| `claude-3-opus-20240229` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 |
-| `claude-3-sonnet-20240229` | ❌ | ❌ | ❌ | 200,000 | 2025-06-20 |
+| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |
+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|
+| `claude-3-5-haiku-20241022` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_5_haiku_20241022_details) |
+| `claude-3-5-sonnet-20241022` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_5_sonnet_20241022_details) |
+| `claude-3-7-sonnet-20250219` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_7_sonnet_20250219_details) |
+| `claude-sonnet-4-20250514` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_sonnet_4_20250514_details) |
+| `claude-opus-4-20250514` | ✅ | ✅ | ⚠️ | 200,000 | 2025-06-20 | [View](#anthropic_claude_opus_4_20250514_details) |
+| `claude-3-5-sonnet-20240620` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_5_sonnet_20240620_details) |
+| `claude-3-haiku-20240307` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_haiku_20240307_details) |
+| `claude-3-opus-20240229` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_opus_20240229_details) |
+| `claude-3-sonnet-20240229` | ❌ | ❌ | ❌ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_sonnet_20240229_details) |
-### Detailed Test Results
-#### claude-3-5-haiku-20241022
+## openai
+
+| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |
+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|
+| `gpt-4.1` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_details) |
+| `gpt-4.1-2025-04-14` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_2025_04_14_details) |
+| `gpt-4.1-nano-2025-04-14` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_nano_2025_04_14_details) |
+| `gpt-4o` | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_details) |
+| `gpt-4o-2024-05-13` | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_2024_05_13_details) |
+| `gpt-4-turbo` | ✅ | ✅ | ⚠️ | 8,192 | 2025-06-20 | [View](#openai_gpt_4_turbo_details) |
+| `gpt-4.1-mini` | ✅ | ✅ | ⚠️ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_mini_details) |
+| `gpt-4.5-preview` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | [View](#openai_gpt_4.5_preview_details) |
+| `gpt-4.5-preview-2025-02-27` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | [View](#openai_gpt_4.5_preview_2025_02_27_details) |
+| `gpt-4o-2024-08-06` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_2024_08_06_details) |
+| `gpt-4-0613` | ✅ | ✅ | ❌ | 8,192 | 2025-06-20 | [View](#openai_gpt_4_0613_details) |
+| `gpt-4-1106-preview` | ✅ | ✅ | ❌ | 128,000 | 2025-06-20 | [View](#openai_gpt_4_1106_preview_details) |
+| `gpt-4-turbo-2024-04-09` | ✅ | ⚠️ | ✅ | 128,000 | 2025-06-20 | [View](#openai_gpt_4_turbo_2024_04_09_details) |
+| `gpt-4.1-mini-2025-04-14` | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_mini_2025_04_14_details) |
+| `gpt-4.1-nano` | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_nano_details) |
+| `gpt-4o-2024-11-20` | ⚠️ | ✅ | ✅ | 8,192 | 2025-06-20 | [View](#openai_gpt_4o_2024_11_20_details) |
+| `gpt-4-turbo-preview` | ✅ | ⚠️ | ❌ | 128,000 | 2025-06-20 | [View](#openai_gpt_4_turbo_preview_details) |
+| `gpt-4-0125-preview` | ⚠️ | ✅ | ❌ | 128,000 | 2025-06-20 | [View](#openai_gpt_4_0125_preview_details) |
+| `gpt-4o-mini` | ⚠️ | ⚠️ | ⚠️ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_mini_details) |
+| `gpt-4o-mini-2024-07-18` | ⚠️ | ⚠️ | ❌ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_mini_2024_07_18_details) |
+| `gpt-4` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_gpt_4_details) |
+| `o1` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o1_details) |
+| `o1-2024-12-17` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o1_2024_12_17_details) |
+| `o3` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_details) |
+| `o3-2025-04-16` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_2025_04_16_details) |
+| `o3-mini` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_mini_details) |
+| `o3-mini-2025-01-31` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_mini_2025_01_31_details) |
+| `o3-pro` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_pro_details) |
+| `o3-pro-2025-06-10` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_pro_2025_06_10_details) |
+
+
+## google_ai
+
+| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |
+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|
+| `gemini-1.5-pro` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_pro_details) |
+| `gemini-1.5-pro-002` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_pro_002_details) |
+| `gemini-1.5-pro-latest` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_pro_latest_details) |
+| `gemini-2.5-flash-preview-04-17-thinking` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_flash_preview_04_17_thinking_details) |
+| `gemini-2.5-pro-preview-03-25` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_preview_03_25_details) |
+| `gemini-2.5-pro-preview-05-06` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_preview_05_06_details) |
+| `gemini-2.5-flash-preview-05-20` | ✅ | ⚠️ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_flash_preview_05_20_details) |
+| `gemini-2.0-flash-thinking-exp` | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_thinking_exp_details) |
+| `gemini-2.0-flash-thinking-exp-1219` | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_thinking_exp_1219_details) |
+| `gemini-2.0-flash-thinking-exp-01-21` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_thinking_exp_01_21_details) |
+| `gemini-2.5-flash-preview-04-17` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_flash_preview_04_17_details) |
+| `gemini-2.5-pro-preview-06-05` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_preview_06_05_details) |
+| `gemini-1.0-pro-vision-latest` | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | [View](#google_ai_gemini_1.0_pro_vision_latest_details) |
+| `gemini-1.5-flash` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_details) |
+| `gemini-1.5-flash-002` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_002_details) |
+| `gemini-1.5-flash-8b` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_8b_details) |
+| `gemini-1.5-flash-8b-001` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_8b_001_details) |
+| `gemini-1.5-flash-8b-latest` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_8b_latest_details) |
+| `gemini-1.5-flash-latest` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_latest_details) |
+| `gemini-2.0-flash` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_details) |
+| `gemini-2.0-flash-001` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_001_details) |
+| `gemini-2.0-flash-exp` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_exp_details) |
+| `gemini-2.0-flash-exp-image-generation` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_exp_image_generation_details) |
+| `gemini-2.0-flash-lite` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_lite_details) |
+| `gemini-2.0-flash-lite-001` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_lite_001_details) |
+| `gemini-2.0-flash-lite-preview` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_lite_preview_details) |
+| `gemini-2.0-flash-lite-preview-02-05` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_lite_preview_02_05_details) |
+| `gemini-2.0-flash-preview-image-generation` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_preview_image_generation_details) |
+| `gemini-2.0-pro-exp` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_pro_exp_details) |
+| `gemini-2.0-pro-exp-02-05` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_pro_exp_02_05_details) |
+| `gemini-2.5-flash-preview-tts` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#google_ai_gemini_2.5_flash_preview_tts_details) |
+| `gemini-2.5-pro-exp-03-25` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_exp_03_25_details) |
+| `gemini-2.5-pro-preview-tts` | ❌ | ❌ | ❌ | 65,536 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_preview_tts_details) |
+| `gemini-exp-1206` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_exp_1206_details) |
+| `gemini-pro-vision` | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | [View](#google_ai_gemini_pro_vision_details) |
+
+
+## letta
+
+| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |
+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|
+| `letta-free` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#letta_letta_free_details) |
+
+
+## together
+
+| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |
+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|
+| `Qwen/Qwen2.5-72B-Instruct-Turbo` | ✅ | ✅ | ⚠️ | 131,072 | 2025-06-20 | [View](#together_qwen_qwen2.5_72b_instruct_turbo_details) |
+| `arcee-ai/virtuoso-large` | ⚠️ | ✅ | ✅ | 131,072 | 2025-06-20 | [View](#together_arcee_ai_virtuoso_large_details) |
+| `Qwen/QwQ-32B` | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | [View](#together_qwen_qwq_32b_details) |
+| `Qwen/Qwen2.5-7B-Instruct-Turbo` | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | [View](#together_qwen_qwen2.5_7b_instruct_turbo_details) |
+| `Qwen/Qwen2.5-Coder-32B-Instruct` | ⚠️ | ✅ | ⚠️ | 16,384 | 2025-06-20 | [View](#together_qwen_qwen2.5_coder_32b_instruct_details) |
+| `arcee-ai/coder-large` | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | [View](#together_arcee_ai_coder_large_details) |
+| `arcee_ai/arcee-spotlight` | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | [View](#together_arcee_ai_arcee_spotlight_details) |
+| `meta-llama/Llama-3.2-3B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.2_3b_instruct_turbo_details) |
+| `meta-llama/Llama-3.3-70B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.3_70b_instruct_turbo_details) |
+| `meta-llama/Llama-3.3-70B-Instruct-Turbo-Free` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.3_70b_instruct_turbo_free_details) |
+| `meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 130,815 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details) |
+| `meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details) |
+| `nvidia/Llama-3.1-Nemotron-70B-Instruct-HF` | ⚠️ | ✅ | ❌ | 32,768 | 2025-06-20 | [View](#together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details) |
+| `arcee-ai/virtuoso-medium-v2` | ⚠️ | ⚠️ | ✅ | 131,072 | 2025-06-20 | [View](#together_arcee_ai_virtuoso_medium_v2_details) |
+| `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8` | ⚠️ | ❌ | ✅ | 1,048,576 | 2025-06-20 | [View](#together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details) |
+| `Qwen/Qwen3-235B-A22B-fp8-tput` | ⚠️ | ⚠️ | ❌ | 40,960 | 2025-06-20 | [View](#together_qwen_qwen3_235b_a22b_fp8_tput_details) |
+| `deepseek-ai/DeepSeek-V3` | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_v3_details) |
+| `meta-llama/Llama-4-Scout-17B-16E-Instruct` | ⚠️ | ⚠️ | ❌ | 1,048,576 | 2025-06-20 | [View](#together_meta_llama_llama_4_scout_17b_16e_instruct_details) |
+| `meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo` | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details) |
+| `mistralai/Mixtral-8x7B-Instruct-v0.1` | ⚠️ | ⚠️ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mixtral_8x7b_instruct_v0.1_details) |
+| `arcee-ai/caller` | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | [View](#together_arcee_ai_caller_details) |
+| `mistralai/Mistral-Small-24B-Instruct-2501` | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mistral_small_24b_instruct_2501_details) |
+| `NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details) |
+| `Qwen/Qwen2-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_qwen_qwen2_72b_instruct_details) |
+| `Qwen/Qwen2-VL-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_qwen_qwen2_vl_72b_instruct_details) |
+| `Qwen/Qwen2.5-VL-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_qwen_qwen2.5_vl_72b_instruct_details) |
+| `arcee-ai/arcee-blitz` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_arcee_ai_arcee_blitz_details) |
+| `arcee-ai/maestro-reasoning` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_arcee_ai_maestro_reasoning_details) |
+| `deepseek-ai/DeepSeek-R1` | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_details) |
+| `deepseek-ai/DeepSeek-R1-Distill-Llama-70B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_distill_llama_70b_details) |
+| `deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details) |
+| `deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details) |
+| `deepseek-ai/DeepSeek-R1-Distill-Qwen-14B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_distill_qwen_14b_details) |
+| `deepseek-ai/DeepSeek-V3-p-dp` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_v3_p_dp_details) |
+| `google/gemma-2-27b-it` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_google_gemma_2_27b_it_details) |
+| `lgai/exaone-3-5-32b-instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_lgai_exaone_3_5_32b_instruct_details) |
+| `lgai/exaone-deep-32b` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_lgai_exaone_deep_32b_details) |
+| `marin-community/marin-8b-instruct` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_marin_community_marin_8b_instruct_details) |
+| `meta-llama/Llama-3-70b-chat-hf` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_meta_llama_llama_3_70b_chat_hf_details) |
+| `meta-llama/Llama-3-8b-chat-hf` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_meta_llama_llama_3_8b_chat_hf_details) |
+| `meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details) |
+| `meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details) |
+| `meta-llama/Llama-Vision-Free` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_vision_free_details) |
+| `meta-llama/Meta-Llama-3-70B-Instruct-Turbo` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3_70b_instruct_turbo_details) |
+| `meta-llama/Meta-Llama-3-8B-Instruct-Lite` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3_8b_instruct_lite_details) |
+| `mistralai/Mistral-7B-Instruct-v0.1` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mistral_7b_instruct_v0.1_details) |
+| `mistralai/Mistral-7B-Instruct-v0.2` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mistral_7b_instruct_v0.2_details) |
+| `mistralai/Mistral-7B-Instruct-v0.3` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mistral_7b_instruct_v0.3_details) |
+| `perplexity-ai/r1-1776` | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | [View](#together_perplexity_ai_r1_1776_details) |
+| `scb10x/scb10x-llama3-1-typhoon2-70b-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details) |
+| `scb10x/scb10x-typhoon-2-1-gemma3-12b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details) |
+| `togethercomputer/MoA-1` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_togethercomputer_moa_1_details) |
+| `togethercomputer/MoA-1-Turbo` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_togethercomputer_moa_1_turbo_details) |
+| `togethercomputer/Refuel-Llm-V2` | ❌ | ❌ | ❌ | 16,384 | 2025-06-20 | [View](#together_togethercomputer_refuel_llm_v2_details) |
+| `togethercomputer/Refuel-Llm-V2-Small` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_togethercomputer_refuel_llm_v2_small_details) |
+
+
+## deepseek
+
+| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |
+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|
+| `deepseek-chat` | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | [View](#deepseek_deepseek_chat_details) |
+| `deepseek-reasoner` | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | [View](#deepseek_deepseek_reasoner_details) |
+
+
+## groq
+
+| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |
+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|
+| `allam-2-7b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_allam_2_7b_details) |
+| `compound-beta` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_compound_beta_details) |
+| `compound-beta-mini` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_compound_beta_mini_details) |
+| `deepseek-r1-distill-llama-70b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_deepseek_r1_distill_llama_70b_details) |
+| `distil-whisper-large-v3-en` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_distil_whisper_large_v3_en_details) |
+| `gemma2-9b-it` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_gemma2_9b_it_details) |
+| `llama-3.1-8b-instant` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama_3.1_8b_instant_details) |
+| `llama-3.3-70b-versatile` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama_3.3_70b_versatile_details) |
+| `llama-guard-3-8b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama_guard_3_8b_details) |
+| `llama3-70b-8192` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama3_70b_8192_details) |
+| `llama3-8b-8192` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama3_8b_8192_details) |
+| `meta-llama/llama-4-maverick-17b-128e-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_4_maverick_17b_128e_instruct_details) |
+| `meta-llama/llama-4-scout-17b-16e-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_4_scout_17b_16e_instruct_details) |
+| `meta-llama/llama-guard-4-12b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_guard_4_12b_details) |
+| `meta-llama/llama-prompt-guard-2-22m` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_prompt_guard_2_22m_details) |
+| `meta-llama/llama-prompt-guard-2-86m` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_prompt_guard_2_86m_details) |
+| `mistral-saba-24b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_mistral_saba_24b_details) |
+| `playai-tts` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_playai_tts_details) |
+| `playai-tts-arabic` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_playai_tts_arabic_details) |
+| `qwen-qwq-32b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_qwen_qwq_32b_details) |
+| `qwen/qwen3-32b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_qwen_qwen3_32b_details) |
+| `whisper-large-v3` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_whisper_large_v3_details) |
+| `whisper-large-v3-turbo` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_whisper_large_v3_turbo_details) |
+
+
+---
+
+# Detailed Test Results
+
+## anthropic
+
+<a id="anthropic_claude_3_5_haiku_20241022_details"></a>
+### claude-3-5-haiku-20241022
### Basic
@@ -55,7 +242,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### claude-3-5-sonnet-20241022
+<a id="anthropic_claude_3_5_sonnet_20241022_details"></a>
+### claude-3-5-sonnet-20241022
### Basic
@@ -82,7 +270,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### claude-3-7-sonnet-20250219
+<a id="anthropic_claude_3_7_sonnet_20250219_details"></a>
+### claude-3-7-sonnet-20250219
### Basic
@@ -109,7 +298,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### claude-sonnet-4-20250514
+<a id="anthropic_claude_sonnet_4_20250514_details"></a>
+### claude-sonnet-4-20250514
### Basic
@@ -136,7 +326,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### claude-opus-4-20250514
+<a id="anthropic_claude_opus_4_20250514_details"></a>
+### claude-opus-4-20250514
### Basic
@@ -163,7 +354,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### claude-3-5-sonnet-20240620
+<a id="anthropic_claude_3_5_sonnet_20240620_details"></a>
+### claude-3-5-sonnet-20240620
### Basic
@@ -190,7 +382,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### claude-3-haiku-20240307
+<a id="anthropic_claude_3_haiku_20240307_details"></a>
+### claude-3-haiku-20240307
### Basic
@@ -217,7 +410,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### claude-3-opus-20240229
+<a id="anthropic_claude_3_opus_20240229_details"></a>
+### claude-3-opus-20240229
### Basic
@@ -244,7 +438,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### claude-3-sonnet-20240229
+<a id="anthropic_claude_3_sonnet_20240229_details"></a>
+### claude-3-sonnet-20240229
### Basic
@@ -272,45 +467,11 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
----
## openai
-| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned |
-|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|
-| `gpt-4.1` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 |
-| `gpt-4.1-2025-04-14` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 |
-| `gpt-4.1-nano-2025-04-14` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 |
-| `gpt-4o` | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 |
-| `gpt-4o-2024-05-13` | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 |
-| `gpt-4-turbo` | ✅ | ✅ | ⚠️ | 8,192 | 2025-06-20 |
-| `gpt-4.1-mini` | ✅ | ✅ | ⚠️ | 1,047,576 | 2025-06-20 |
-| `gpt-4.5-preview` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 |
-| `gpt-4.5-preview-2025-02-27` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 |
-| `gpt-4o-2024-08-06` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 |
-| `gpt-4-0613` | ✅ | ✅ | ❌ | 8,192 | 2025-06-20 |
-| `gpt-4-1106-preview` | ✅ | ✅ | ❌ | 128,000 | 2025-06-20 |
-| `gpt-4-turbo-2024-04-09` | ✅ | ⚠️ | ✅ | 128,000 | 2025-06-20 |
-| `gpt-4.1-mini-2025-04-14` | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 |
-| `gpt-4.1-nano` | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 |
-| `gpt-4o-2024-11-20` | ⚠️ | ✅ | ✅ | 8,192 | 2025-06-20 |
-| `gpt-4-turbo-preview` | ✅ | ⚠️ | ❌ | 128,000 | 2025-06-20 |
-| `gpt-4-0125-preview` | ⚠️ | ✅ | ❌ | 128,000 | 2025-06-20 |
-| `gpt-4o-mini` | ⚠️ | ⚠️ | ⚠️ | 128,000 | 2025-06-20 |
-| `gpt-4o-mini-2024-07-18` | ⚠️ | ⚠️ | ❌ | 128,000 | 2025-06-20 |
-| `gpt-4` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `o1` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `o1-2024-12-17` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `o3` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `o3-2025-04-16` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `o3-mini` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `o3-mini-2025-01-31` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `o3-pro` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `o3-pro-2025-06-10` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-
-### Detailed Test Results
-
-#### gpt-4.1
+<a id="openai_gpt_4.1_details"></a>
+### gpt-4.1
### Basic
@@ -337,7 +498,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4.1-2025-04-14
+<a id="openai_gpt_4.1_2025_04_14_details"></a>
+### gpt-4.1-2025-04-14
### Basic
@@ -364,7 +526,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4.1-nano-2025-04-14
+<a id="openai_gpt_4.1_nano_2025_04_14_details"></a>
+### gpt-4.1-nano-2025-04-14
### Basic
@@ -391,7 +554,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4o
+<a id="openai_gpt_4o_details"></a>
+### gpt-4o
### Basic
@@ -418,7 +582,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4o-2024-05-13
+<a id="openai_gpt_4o_2024_05_13_details"></a>
+### gpt-4o-2024-05-13
### Basic
@@ -445,7 +610,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4-turbo
+<a id="openai_gpt_4_turbo_details"></a>
+### gpt-4-turbo
### Basic
@@ -472,7 +638,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4.1-mini
+<a id="openai_gpt_4.1_mini_details"></a>
+### gpt-4.1-mini
### Basic
@@ -499,7 +666,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4.5-preview
+<a id="openai_gpt_4.5_preview_details"></a>
+### gpt-4.5-preview
### Basic
@@ -526,7 +694,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4.5-preview-2025-02-27
+<a id="openai_gpt_4.5_preview_2025_02_27_details"></a>
+### gpt-4.5-preview-2025-02-27
### Basic
@@ -553,7 +722,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4o-2024-08-06
+<a id="openai_gpt_4o_2024_08_06_details"></a>
+### gpt-4o-2024-08-06
### Basic
@@ -580,7 +750,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4-0613
+<a id="openai_gpt_4_0613_details"></a>
+### gpt-4-0613
### Basic
@@ -607,7 +778,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4-1106-preview
+<a id="openai_gpt_4_1106_preview_details"></a>
+### gpt-4-1106-preview
### Basic
@@ -634,7 +806,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4-turbo-2024-04-09
+<a id="openai_gpt_4_turbo_2024_04_09_details"></a>
+### gpt-4-turbo-2024-04-09
### Basic
@@ -661,7 +834,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4.1-mini-2025-04-14
+<a id="openai_gpt_4.1_mini_2025_04_14_details"></a>
+### gpt-4.1-mini-2025-04-14
### Basic
@@ -688,7 +862,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4.1-nano
+<a id="openai_gpt_4.1_nano_details"></a>
+### gpt-4.1-nano
### Basic
@@ -715,7 +890,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4o-2024-11-20
+<a id="openai_gpt_4o_2024_11_20_details"></a>
+### gpt-4o-2024-11-20
### Basic
@@ -742,7 +918,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4-turbo-preview
+<a id="openai_gpt_4_turbo_preview_details"></a>
+### gpt-4-turbo-preview
### Basic
@@ -769,7 +946,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4-0125-preview
+<a id="openai_gpt_4_0125_preview_details"></a>
+### gpt-4-0125-preview
### Basic
@@ -796,7 +974,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4o-mini
+<a id="openai_gpt_4o_mini_details"></a>
+### gpt-4o-mini
### Basic
@@ -823,7 +1002,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4o-mini-2024-07-18
+<a id="openai_gpt_4o_mini_2024_07_18_details"></a>
+### gpt-4o-mini-2024-07-18
### Basic
@@ -850,7 +1030,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gpt-4
+<a id="openai_gpt_4_details"></a>
+### gpt-4
### Basic
@@ -877,7 +1058,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### o1
+<a id="openai_o1_details"></a>
+### o1
### Basic
@@ -904,7 +1086,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### o1-2024-12-17
+<a id="openai_o1_2024_12_17_details"></a>
+### o1-2024-12-17
### Basic
@@ -931,7 +1114,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### o3
+<a id="openai_o3_details"></a>
+### o3
### Basic
@@ -958,7 +1142,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### o3-2025-04-16
+<a id="openai_o3_2025_04_16_details"></a>
+### o3-2025-04-16
### Basic
@@ -985,7 +1170,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### o3-mini
+<a id="openai_o3_mini_details"></a>
+### o3-mini
### Basic
@@ -1012,7 +1198,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### o3-mini-2025-01-31
+<a id="openai_o3_mini_2025_01_31_details"></a>
+### o3-mini-2025-01-31
### Basic
@@ -1039,7 +1226,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### o3-pro
+<a id="openai_o3_pro_details"></a>
+### o3-pro
### Basic
@@ -1066,7 +1254,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### o3-pro-2025-06-10
+<a id="openai_o3_pro_2025_06_10_details"></a>
+### o3-pro-2025-06-10
### Basic
@@ -1094,51 +1283,11 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
----
## google_ai
-| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned |
-|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|
-| `gemini-1.5-pro` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 |
-| `gemini-1.5-pro-002` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 |
-| `gemini-1.5-pro-latest` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 |
-| `gemini-2.5-flash-preview-04-17-thinking` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 |
-| `gemini-2.5-pro-preview-03-25` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 |
-| `gemini-2.5-pro-preview-05-06` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 |
-| `gemini-2.5-flash-preview-05-20` | ✅ | ⚠️ | ✅ | 1,048,576 | 2025-06-20 |
-| `gemini-2.0-flash-thinking-exp` | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 |
-| `gemini-2.0-flash-thinking-exp-1219` | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 |
-| `gemini-2.0-flash-thinking-exp-01-21` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 |
-| `gemini-2.5-flash-preview-04-17` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 |
-| `gemini-2.5-pro-preview-06-05` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 |
-| `gemini-1.0-pro-vision-latest` | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 |
-| `gemini-1.5-flash` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 |
-| `gemini-1.5-flash-002` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 |
-| `gemini-1.5-flash-8b` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 |
-| `gemini-1.5-flash-8b-001` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 |
-| `gemini-1.5-flash-8b-latest` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 |
-| `gemini-1.5-flash-latest` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 |
-| `gemini-2.0-flash` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |
-| `gemini-2.0-flash-001` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |
-| `gemini-2.0-flash-exp` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |
-| `gemini-2.0-flash-exp-image-generation` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |
-| `gemini-2.0-flash-lite` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |
-| `gemini-2.0-flash-lite-001` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |
-| `gemini-2.0-flash-lite-preview` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |
-| `gemini-2.0-flash-lite-preview-02-05` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |
-| `gemini-2.0-flash-preview-image-generation` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |
-| `gemini-2.0-pro-exp` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |
-| `gemini-2.0-pro-exp-02-05` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |
-| `gemini-2.5-flash-preview-tts` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |
-| `gemini-2.5-pro-exp-03-25` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |
-| `gemini-2.5-pro-preview-tts` | ❌ | ❌ | ❌ | 65,536 | 2025-06-20 |
-| `gemini-exp-1206` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |
-| `gemini-pro-vision` | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 |
-
-### Detailed Test Results
-
-#### gemini-1.5-pro
+<a id="google_ai_gemini_1.5_pro_details"></a>
+### gemini-1.5-pro
### Basic
@@ -1165,7 +1314,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-1.5-pro-002
+<a id="google_ai_gemini_1.5_pro_002_details"></a>
+### gemini-1.5-pro-002
### Basic
@@ -1192,7 +1342,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-1.5-pro-latest
+<a id="google_ai_gemini_1.5_pro_latest_details"></a>
+### gemini-1.5-pro-latest
### Basic
@@ -1219,7 +1370,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.5-flash-preview-04-17-thinking
+<a id="google_ai_gemini_2.5_flash_preview_04_17_thinking_details"></a>
+### gemini-2.5-flash-preview-04-17-thinking
### Basic
@@ -1246,7 +1398,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.5-pro-preview-03-25
+<a id="google_ai_gemini_2.5_pro_preview_03_25_details"></a>
+### gemini-2.5-pro-preview-03-25
### Basic
@@ -1273,7 +1426,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.5-pro-preview-05-06
+<a id="google_ai_gemini_2.5_pro_preview_05_06_details"></a>
+### gemini-2.5-pro-preview-05-06
### Basic
@@ -1300,7 +1454,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.5-flash-preview-05-20
+<a id="google_ai_gemini_2.5_flash_preview_05_20_details"></a>
+### gemini-2.5-flash-preview-05-20
### Basic
@@ -1327,7 +1482,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.0-flash-thinking-exp
+<a id="google_ai_gemini_2.0_flash_thinking_exp_details"></a>
+### gemini-2.0-flash-thinking-exp
### Basic
@@ -1354,7 +1510,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.0-flash-thinking-exp-1219
+<a id="google_ai_gemini_2.0_flash_thinking_exp_1219_details"></a>
+### gemini-2.0-flash-thinking-exp-1219
### Basic
@@ -1381,7 +1538,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.0-flash-thinking-exp-01-21
+<a id="google_ai_gemini_2.0_flash_thinking_exp_01_21_details"></a>
+### gemini-2.0-flash-thinking-exp-01-21
### Basic
@@ -1408,7 +1566,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.5-flash-preview-04-17
+<a id="google_ai_gemini_2.5_flash_preview_04_17_details"></a>
+### gemini-2.5-flash-preview-04-17
### Basic
@@ -1435,7 +1594,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.5-pro-preview-06-05
+<a id="google_ai_gemini_2.5_pro_preview_06_05_details"></a>
+### gemini-2.5-pro-preview-06-05
### Basic
@@ -1462,7 +1622,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-1.0-pro-vision-latest
+<a id="google_ai_gemini_1.0_pro_vision_latest_details"></a>
+### gemini-1.0-pro-vision-latest
### Basic
@@ -1489,7 +1650,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-1.5-flash
+<a id="google_ai_gemini_1.5_flash_details"></a>
+### gemini-1.5-flash
### Basic
@@ -1516,7 +1678,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-1.5-flash-002
+<a id="google_ai_gemini_1.5_flash_002_details"></a>
+### gemini-1.5-flash-002
### Basic
@@ -1543,7 +1706,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-1.5-flash-8b
+<a id="google_ai_gemini_1.5_flash_8b_details"></a>
+### gemini-1.5-flash-8b
### Basic
@@ -1570,7 +1734,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-1.5-flash-8b-001
+<a id="google_ai_gemini_1.5_flash_8b_001_details"></a>
+### gemini-1.5-flash-8b-001
### Basic
@@ -1597,7 +1762,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-1.5-flash-8b-latest
+<a id="google_ai_gemini_1.5_flash_8b_latest_details"></a>
+### gemini-1.5-flash-8b-latest
### Basic
@@ -1624,7 +1790,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-1.5-flash-latest
+<a id="google_ai_gemini_1.5_flash_latest_details"></a>
+### gemini-1.5-flash-latest
### Basic
@@ -1651,7 +1818,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.0-flash
+<a id="google_ai_gemini_2.0_flash_details"></a>
+### gemini-2.0-flash
### Basic
@@ -1678,7 +1846,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.0-flash-001
+<a id="google_ai_gemini_2.0_flash_001_details"></a>
+### gemini-2.0-flash-001
### Basic
@@ -1705,7 +1874,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.0-flash-exp
+<a id="google_ai_gemini_2.0_flash_exp_details"></a>
+### gemini-2.0-flash-exp
### Basic
@@ -1732,7 +1902,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.0-flash-exp-image-generation
+<a id="google_ai_gemini_2.0_flash_exp_image_generation_details"></a>
+### gemini-2.0-flash-exp-image-generation
### Basic
@@ -1759,7 +1930,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.0-flash-lite
+<a id="google_ai_gemini_2.0_flash_lite_details"></a>
+### gemini-2.0-flash-lite
### Basic
@@ -1786,7 +1958,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.0-flash-lite-001
+<a id="google_ai_gemini_2.0_flash_lite_001_details"></a>
+### gemini-2.0-flash-lite-001
### Basic
@@ -1813,7 +1986,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.0-flash-lite-preview
+<a id="google_ai_gemini_2.0_flash_lite_preview_details"></a>
+### gemini-2.0-flash-lite-preview
### Basic
@@ -1840,7 +2014,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.0-flash-lite-preview-02-05
+<a id="google_ai_gemini_2.0_flash_lite_preview_02_05_details"></a>
+### gemini-2.0-flash-lite-preview-02-05
### Basic
@@ -1867,7 +2042,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.0-flash-preview-image-generation
+<a id="google_ai_gemini_2.0_flash_preview_image_generation_details"></a>
+### gemini-2.0-flash-preview-image-generation
### Basic
@@ -1894,7 +2070,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.0-pro-exp
+<a id="google_ai_gemini_2.0_pro_exp_details"></a>
+### gemini-2.0-pro-exp
### Basic
@@ -1921,7 +2098,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.0-pro-exp-02-05
+<a id="google_ai_gemini_2.0_pro_exp_02_05_details"></a>
+### gemini-2.0-pro-exp-02-05
### Basic
@@ -1948,7 +2126,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.5-flash-preview-tts
+<a id="google_ai_gemini_2.5_flash_preview_tts_details"></a>
+### gemini-2.5-flash-preview-tts
### Basic
@@ -1975,7 +2154,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.5-pro-exp-03-25
+<a id="google_ai_gemini_2.5_pro_exp_03_25_details"></a>
+### gemini-2.5-pro-exp-03-25
### Basic
@@ -2002,7 +2182,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-2.5-pro-preview-tts
+<a id="google_ai_gemini_2.5_pro_preview_tts_details"></a>
+### gemini-2.5-pro-preview-tts
### Basic
@@ -2029,7 +2210,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-exp-1206
+<a id="google_ai_gemini_exp_1206_details"></a>
+### gemini-exp-1206
### Basic
@@ -2056,7 +2238,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemini-pro-vision
+<a id="google_ai_gemini_pro_vision_details"></a>
+### gemini-pro-vision
### Basic
@@ -2084,17 +2267,11 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
----
## letta
-| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned |
-|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|
-| `letta-free` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 |
-
-### Detailed Test Results
-
-#### letta-free
+<a id="letta_letta_free_details"></a>
+### letta-free
### Basic
@@ -2122,71 +2299,11 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
----
## together
-| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned |
-|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|
-| `Qwen/Qwen2.5-72B-Instruct-Turbo` | ✅ | ✅ | ⚠️ | 131,072 | 2025-06-20 |
-| `arcee-ai/virtuoso-large` | ⚠️ | ✅ | ✅ | 131,072 | 2025-06-20 |
-| `Qwen/QwQ-32B` | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 |
-| `Qwen/Qwen2.5-7B-Instruct-Turbo` | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 |
-| `Qwen/Qwen2.5-Coder-32B-Instruct` | ⚠️ | ✅ | ⚠️ | 16,384 | 2025-06-20 |
-| `arcee-ai/coder-large` | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 |
-| `arcee_ai/arcee-spotlight` | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 |
-| `meta-llama/Llama-3.2-3B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 |
-| `meta-llama/Llama-3.3-70B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 |
-| `meta-llama/Llama-3.3-70B-Instruct-Turbo-Free` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 |
-| `meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 130,815 | 2025-06-20 |
-| `meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 |
-| `nvidia/Llama-3.1-Nemotron-70B-Instruct-HF` | ⚠️ | ✅ | ❌ | 32,768 | 2025-06-20 |
-| `arcee-ai/virtuoso-medium-v2` | ⚠️ | ⚠️ | ✅ | 131,072 | 2025-06-20 |
-| `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8` | ⚠️ | ❌ | ✅ | 1,048,576 | 2025-06-20 |
-| `Qwen/Qwen3-235B-A22B-fp8-tput` | ⚠️ | ⚠️ | ❌ | 40,960 | 2025-06-20 |
-| `deepseek-ai/DeepSeek-V3` | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 |
-| `meta-llama/Llama-4-Scout-17B-16E-Instruct` | ⚠️ | ⚠️ | ❌ | 1,048,576 | 2025-06-20 |
-| `meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo` | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 |
-| `mistralai/Mixtral-8x7B-Instruct-v0.1` | ⚠️ | ⚠️ | ❌ | 32,768 | 2025-06-20 |
-| `arcee-ai/caller` | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 |
-| `mistralai/Mistral-Small-24B-Instruct-2501` | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 |
-| `NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |
-| `Qwen/Qwen2-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |
-| `Qwen/Qwen2-VL-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |
-| `Qwen/Qwen2.5-VL-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |
-| `arcee-ai/arcee-blitz` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |
-| `arcee-ai/maestro-reasoning` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |
-| `deepseek-ai/DeepSeek-R1` | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 |
-| `deepseek-ai/DeepSeek-R1-Distill-Llama-70B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |
-| `deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |
-| `deepseek-ai/DeepSeek-R1-Distill-Qwen-14B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |
-| `deepseek-ai/DeepSeek-V3-p-dp` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |
-| `google/gemma-2-27b-it` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `lgai/exaone-3-5-32b-instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |
-| `lgai/exaone-deep-32b` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |
-| `marin-community/marin-8b-instruct` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |
-| `meta-llama/Llama-3-70b-chat-hf` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `meta-llama/Llama-3-8b-chat-hf` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |
-| `meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |
-| `meta-llama/Llama-Vision-Free` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |
-| `meta-llama/Meta-Llama-3-70B-Instruct-Turbo` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `meta-llama/Meta-Llama-3-8B-Instruct-Lite` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `mistralai/Mistral-7B-Instruct-v0.1` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |
-| `mistralai/Mistral-7B-Instruct-v0.2` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |
-| `mistralai/Mistral-7B-Instruct-v0.3` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |
-| `perplexity-ai/r1-1776` | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 |
-| `scb10x/scb10x-llama3-1-typhoon2-70b-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `scb10x/scb10x-typhoon-2-1-gemma3-12b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `togethercomputer/MoA-1` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |
-| `togethercomputer/MoA-1-Turbo` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |
-| `togethercomputer/Refuel-Llm-V2` | ❌ | ❌ | ❌ | 16,384 | 2025-06-20 |
-| `togethercomputer/Refuel-Llm-V2-Small` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-
-### Detailed Test Results
-
-#### Qwen/Qwen2.5-72B-Instruct-Turbo
+<a id="together_qwen_qwen2.5_72b_instruct_turbo_details"></a>
+### Qwen/Qwen2.5-72B-Instruct-Turbo
### Basic
@@ -2213,7 +2330,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### arcee-ai/virtuoso-large
+<a id="together_arcee_ai_virtuoso_large_details"></a>
+### arcee-ai/virtuoso-large
### Basic
@@ -2240,7 +2358,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### Qwen/QwQ-32B
+<a id="together_qwen_qwq_32b_details"></a>
+### Qwen/QwQ-32B
### Basic
@@ -2267,7 +2386,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### Qwen/Qwen2.5-7B-Instruct-Turbo
+<a id="together_qwen_qwen2.5_7b_instruct_turbo_details"></a>
+### Qwen/Qwen2.5-7B-Instruct-Turbo
### Basic
@@ -2294,7 +2414,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### Qwen/Qwen2.5-Coder-32B-Instruct
+<a id="together_qwen_qwen2.5_coder_32b_instruct_details"></a>
+### Qwen/Qwen2.5-Coder-32B-Instruct
### Basic
@@ -2321,7 +2442,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### arcee-ai/coder-large
+<a id="together_arcee_ai_coder_large_details"></a>
+### arcee-ai/coder-large
### Basic
@@ -2348,7 +2470,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### arcee_ai/arcee-spotlight
+<a id="together_arcee_ai_arcee_spotlight_details"></a>
+### arcee_ai/arcee-spotlight
### Basic
@@ -2375,7 +2498,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/Llama-3.2-3B-Instruct-Turbo
+<a id="together_meta_llama_llama_3.2_3b_instruct_turbo_details"></a>
+### meta-llama/Llama-3.2-3B-Instruct-Turbo
### Basic
@@ -2402,7 +2526,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/Llama-3.3-70B-Instruct-Turbo
+<a id="together_meta_llama_llama_3.3_70b_instruct_turbo_details"></a>
+### meta-llama/Llama-3.3-70B-Instruct-Turbo
### Basic
@@ -2429,7 +2554,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/Llama-3.3-70B-Instruct-Turbo-Free
+<a id="together_meta_llama_llama_3.3_70b_instruct_turbo_free_details"></a>
+### meta-llama/Llama-3.3-70B-Instruct-Turbo-Free
### Basic
@@ -2456,7 +2582,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo
+<a id="together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details"></a>
+### meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo
### Basic
@@ -2483,7 +2610,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo
+<a id="together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details"></a>
+### meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo
### Basic
@@ -2510,7 +2638,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### nvidia/Llama-3.1-Nemotron-70B-Instruct-HF
+<a id="together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details"></a>
+### nvidia/Llama-3.1-Nemotron-70B-Instruct-HF
### Basic
@@ -2537,7 +2666,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### arcee-ai/virtuoso-medium-v2
+<a id="together_arcee_ai_virtuoso_medium_v2_details"></a>
+### arcee-ai/virtuoso-medium-v2
### Basic
@@ -2564,7 +2694,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8
+<a id="together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details"></a>
+### meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8
### Basic
@@ -2591,7 +2722,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### Qwen/Qwen3-235B-A22B-fp8-tput
+<a id="together_qwen_qwen3_235b_a22b_fp8_tput_details"></a>
+### Qwen/Qwen3-235B-A22B-fp8-tput
### Basic
@@ -2618,7 +2750,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### deepseek-ai/DeepSeek-V3
+<a id="together_deepseek_ai_deepseek_v3_details"></a>
+### deepseek-ai/DeepSeek-V3
### Basic
@@ -2645,7 +2778,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/Llama-4-Scout-17B-16E-Instruct
+<a id="together_meta_llama_llama_4_scout_17b_16e_instruct_details"></a>
+### meta-llama/Llama-4-Scout-17B-16E-Instruct
### Basic
@@ -2672,7 +2806,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo
+<a id="together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details"></a>
+### meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo
### Basic
@@ -2699,7 +2834,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### mistralai/Mixtral-8x7B-Instruct-v0.1
+<a id="together_mistralai_mixtral_8x7b_instruct_v0.1_details"></a>
+### mistralai/Mixtral-8x7B-Instruct-v0.1
### Basic
@@ -2726,7 +2862,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### arcee-ai/caller
+<a id="together_arcee_ai_caller_details"></a>
+### arcee-ai/caller
### Basic
@@ -2753,7 +2890,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### mistralai/Mistral-Small-24B-Instruct-2501
+<a id="together_mistralai_mistral_small_24b_instruct_2501_details"></a>
+### mistralai/Mistral-Small-24B-Instruct-2501
### Basic
@@ -2780,7 +2918,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO
+<a id="together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details"></a>
+### NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO
### Basic
@@ -2807,7 +2946,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### Qwen/Qwen2-72B-Instruct
+<a id="together_qwen_qwen2_72b_instruct_details"></a>
+### Qwen/Qwen2-72B-Instruct
### Basic
@@ -2834,7 +2974,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### Qwen/Qwen2-VL-72B-Instruct
+<a id="together_qwen_qwen2_vl_72b_instruct_details"></a>
+### Qwen/Qwen2-VL-72B-Instruct
### Basic
@@ -2861,7 +3002,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### Qwen/Qwen2.5-VL-72B-Instruct
+<a id="together_qwen_qwen2.5_vl_72b_instruct_details"></a>
+### Qwen/Qwen2.5-VL-72B-Instruct
### Basic
@@ -2888,7 +3030,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### arcee-ai/arcee-blitz
+<a id="together_arcee_ai_arcee_blitz_details"></a>
+### arcee-ai/arcee-blitz
### Basic
@@ -2915,7 +3058,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### arcee-ai/maestro-reasoning
+<a id="together_arcee_ai_maestro_reasoning_details"></a>
+### arcee-ai/maestro-reasoning
### Basic
@@ -2942,7 +3086,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### deepseek-ai/DeepSeek-R1
+<a id="together_deepseek_ai_deepseek_r1_details"></a>
+### deepseek-ai/DeepSeek-R1
### Basic
@@ -2969,7 +3114,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B
+<a id="together_deepseek_ai_deepseek_r1_distill_llama_70b_details"></a>
+### deepseek-ai/DeepSeek-R1-Distill-Llama-70B
### Basic
@@ -2996,7 +3142,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free
+<a id="together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details"></a>
+### deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free
### Basic
@@ -3023,7 +3170,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
+<a id="together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details"></a>
+### deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
### Basic
@@ -3050,7 +3198,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### deepseek-ai/DeepSeek-R1-Distill-Qwen-14B
+<a id="together_deepseek_ai_deepseek_r1_distill_qwen_14b_details"></a>
+### deepseek-ai/DeepSeek-R1-Distill-Qwen-14B
### Basic
@@ -3077,7 +3226,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### deepseek-ai/DeepSeek-V3-p-dp
+<a id="together_deepseek_ai_deepseek_v3_p_dp_details"></a>
+### deepseek-ai/DeepSeek-V3-p-dp
### Basic
@@ -3104,7 +3254,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### google/gemma-2-27b-it
+<a id="together_google_gemma_2_27b_it_details"></a>
+### google/gemma-2-27b-it
### Basic
@@ -3131,7 +3282,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### lgai/exaone-3-5-32b-instruct
+<a id="together_lgai_exaone_3_5_32b_instruct_details"></a>
+### lgai/exaone-3-5-32b-instruct
### Basic
@@ -3158,7 +3310,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### lgai/exaone-deep-32b
+<a id="together_lgai_exaone_deep_32b_details"></a>
+### lgai/exaone-deep-32b
### Basic
@@ -3185,7 +3338,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### marin-community/marin-8b-instruct
+<a id="together_marin_community_marin_8b_instruct_details"></a>
+### marin-community/marin-8b-instruct
### Basic
@@ -3212,7 +3366,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/Llama-3-70b-chat-hf
+<a id="together_meta_llama_llama_3_70b_chat_hf_details"></a>
+### meta-llama/Llama-3-70b-chat-hf
### Basic
@@ -3239,7 +3394,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/Llama-3-8b-chat-hf
+<a id="together_meta_llama_llama_3_8b_chat_hf_details"></a>
+### meta-llama/Llama-3-8b-chat-hf
### Basic
@@ -3266,7 +3422,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo
+<a id="together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details"></a>
+### meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo
### Basic
@@ -3293,7 +3450,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo
+<a id="together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details"></a>
+### meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo
### Basic
@@ -3320,7 +3478,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/Llama-Vision-Free
+<a id="together_meta_llama_llama_vision_free_details"></a>
+### meta-llama/Llama-Vision-Free
### Basic
@@ -3347,7 +3506,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/Meta-Llama-3-70B-Instruct-Turbo
+<a id="together_meta_llama_meta_llama_3_70b_instruct_turbo_details"></a>
+### meta-llama/Meta-Llama-3-70B-Instruct-Turbo
### Basic
@@ -3374,7 +3534,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/Meta-Llama-3-8B-Instruct-Lite
+<a id="together_meta_llama_meta_llama_3_8b_instruct_lite_details"></a>
+### meta-llama/Meta-Llama-3-8B-Instruct-Lite
### Basic
@@ -3401,7 +3562,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### mistralai/Mistral-7B-Instruct-v0.1
+<a id="together_mistralai_mistral_7b_instruct_v0.1_details"></a>
+### mistralai/Mistral-7B-Instruct-v0.1
### Basic
@@ -3428,7 +3590,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### mistralai/Mistral-7B-Instruct-v0.2
+<a id="together_mistralai_mistral_7b_instruct_v0.2_details"></a>
+### mistralai/Mistral-7B-Instruct-v0.2
### Basic
@@ -3455,7 +3618,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### mistralai/Mistral-7B-Instruct-v0.3
+<a id="together_mistralai_mistral_7b_instruct_v0.3_details"></a>
+### mistralai/Mistral-7B-Instruct-v0.3
### Basic
@@ -3482,7 +3646,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### perplexity-ai/r1-1776
+<a id="together_perplexity_ai_r1_1776_details"></a>
+### perplexity-ai/r1-1776
### Basic
@@ -3509,7 +3674,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### scb10x/scb10x-llama3-1-typhoon2-70b-instruct
+<a id="together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details"></a>
+### scb10x/scb10x-llama3-1-typhoon2-70b-instruct
### Basic
@@ -3536,7 +3702,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### scb10x/scb10x-typhoon-2-1-gemma3-12b
+<a id="together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details"></a>
+### scb10x/scb10x-typhoon-2-1-gemma3-12b
### Basic
@@ -3563,7 +3730,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### togethercomputer/MoA-1
+<a id="together_togethercomputer_moa_1_details"></a>
+### togethercomputer/MoA-1
### Basic
@@ -3590,7 +3758,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### togethercomputer/MoA-1-Turbo
+<a id="together_togethercomputer_moa_1_turbo_details"></a>
+### togethercomputer/MoA-1-Turbo
### Basic
@@ -3617,7 +3786,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### togethercomputer/Refuel-Llm-V2
+<a id="together_togethercomputer_refuel_llm_v2_details"></a>
+### togethercomputer/Refuel-Llm-V2
### Basic
@@ -3644,7 +3814,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### togethercomputer/Refuel-Llm-V2-Small
+<a id="together_togethercomputer_refuel_llm_v2_small_details"></a>
+### togethercomputer/Refuel-Llm-V2-Small
### Basic
@@ -3672,18 +3843,11 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
----
## deepseek
-| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned |
-|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|
-| `deepseek-chat` | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 |
-| `deepseek-reasoner` | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 |
-
-### Detailed Test Results
-
-#### deepseek-chat
+<a id="deepseek_deepseek_chat_details"></a>
+### deepseek-chat
### Basic
@@ -3710,7 +3874,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### deepseek-reasoner
+<a id="deepseek_deepseek_reasoner_details"></a>
+### deepseek-reasoner
### Basic
@@ -3738,39 +3903,11 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
----
## groq
-| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned |
-|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|
-| `allam-2-7b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `compound-beta` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `compound-beta-mini` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `deepseek-r1-distill-llama-70b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `distil-whisper-large-v3-en` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `gemma2-9b-it` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `llama-3.1-8b-instant` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `llama-3.3-70b-versatile` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `llama-guard-3-8b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `llama3-70b-8192` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `llama3-8b-8192` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `meta-llama/llama-4-maverick-17b-128e-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `meta-llama/llama-4-scout-17b-16e-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `meta-llama/llama-guard-4-12b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `meta-llama/llama-prompt-guard-2-22m` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `meta-llama/llama-prompt-guard-2-86m` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `mistral-saba-24b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `playai-tts` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `playai-tts-arabic` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `qwen-qwq-32b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `qwen/qwen3-32b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `whisper-large-v3` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-| `whisper-large-v3-turbo` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |
-
-### Detailed Test Results
-
-#### allam-2-7b
+<a id="groq_allam_2_7b_details"></a>
+### allam-2-7b
### Basic
@@ -3797,7 +3934,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### compound-beta
+<a id="groq_compound_beta_details"></a>
+### compound-beta
### Basic
@@ -3824,7 +3962,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### compound-beta-mini
+<a id="groq_compound_beta_mini_details"></a>
+### compound-beta-mini
### Basic
@@ -3851,7 +3990,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### deepseek-r1-distill-llama-70b
+<a id="groq_deepseek_r1_distill_llama_70b_details"></a>
+### deepseek-r1-distill-llama-70b
### Basic
@@ -3878,7 +4018,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### distil-whisper-large-v3-en
+<a id="groq_distil_whisper_large_v3_en_details"></a>
+### distil-whisper-large-v3-en
### Basic
@@ -3905,7 +4046,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### gemma2-9b-it
+<a id="groq_gemma2_9b_it_details"></a>
+### gemma2-9b-it
### Basic
@@ -3932,7 +4074,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### llama-3.1-8b-instant
+<a id="groq_llama_3.1_8b_instant_details"></a>
+### llama-3.1-8b-instant
### Basic
@@ -3959,7 +4102,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### llama-3.3-70b-versatile
+<a id="groq_llama_3.3_70b_versatile_details"></a>
+### llama-3.3-70b-versatile
### Basic
@@ -3986,7 +4130,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### llama-guard-3-8b
+<a id="groq_llama_guard_3_8b_details"></a>
+### llama-guard-3-8b
### Basic
@@ -4013,7 +4158,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### llama3-70b-8192
+<a id="groq_llama3_70b_8192_details"></a>
+### llama3-70b-8192
### Basic
@@ -4040,7 +4186,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### llama3-8b-8192
+<a id="groq_llama3_8b_8192_details"></a>
+### llama3-8b-8192
### Basic
@@ -4067,7 +4214,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/llama-4-maverick-17b-128e-instruct
+<a id="groq_meta_llama_llama_4_maverick_17b_128e_instruct_details"></a>
+### meta-llama/llama-4-maverick-17b-128e-instruct
### Basic
@@ -4094,7 +4242,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/llama-4-scout-17b-16e-instruct
+<a id="groq_meta_llama_llama_4_scout_17b_16e_instruct_details"></a>
+### meta-llama/llama-4-scout-17b-16e-instruct
### Basic
@@ -4121,7 +4270,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/llama-guard-4-12b
+<a id="groq_meta_llama_llama_guard_4_12b_details"></a>
+### meta-llama/llama-guard-4-12b
### Basic
@@ -4148,7 +4298,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/llama-prompt-guard-2-22m
+<a id="groq_meta_llama_llama_prompt_guard_2_22m_details"></a>
+### meta-llama/llama-prompt-guard-2-22m
### Basic
@@ -4175,7 +4326,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### meta-llama/llama-prompt-guard-2-86m
+<a id="groq_meta_llama_llama_prompt_guard_2_86m_details"></a>
+### meta-llama/llama-prompt-guard-2-86m
### Basic
@@ -4202,7 +4354,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### mistral-saba-24b
+<a id="groq_mistral_saba_24b_details"></a>
+### mistral-saba-24b
### Basic
@@ -4229,7 +4382,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### playai-tts
+<a id="groq_playai_tts_details"></a>
+### playai-tts
### Basic
@@ -4256,7 +4410,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### playai-tts-arabic
+<a id="groq_playai_tts_arabic_details"></a>
+### playai-tts-arabic
### Basic
@@ -4283,7 +4438,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### qwen-qwq-32b
+<a id="groq_qwen_qwq_32b_details"></a>
+### qwen-qwq-32b
### Basic
@@ -4310,7 +4466,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### qwen/qwen3-32b
+<a id="groq_qwen_qwen3_32b_details"></a>
+### qwen/qwen3-32b
### Basic
@@ -4337,7 +4494,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### whisper-large-v3
+<a id="groq_whisper_large_v3_details"></a>
+### whisper-large-v3
### Basic
@@ -4364,7 +4522,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
-#### whisper-large-v3-turbo
+<a id="groq_whisper_large_v3_turbo_details"></a>
+### whisper-large-v3-turbo
### Basic
@@ -4392,4 +4551,3 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025
- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)
----