Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
move results to bottom of the page
Kian Jones684913364d47
diff --git a/.github/scripts/model-sweep/generate_model_sweep_markdown.py b/.github/scripts/model-sweep/generate_model_sweep_markdown.pyindex 09aafa7e5..9d965b576 100644--- a/.github/scripts/model-sweep/generate_model_sweep_markdown.py+++ b/.github/scripts/model-sweep/generate_model_sweep_markdown.py@@ -355,8 +355,9 @@ def process_model_sweep_report(input_file, output_file, config_file=None, debug=all_provider_data[provider_name] = model_dataprovider_support_scores[provider_name] = calculate_provider_support_score(model_data, list(feature_mapping.keys()))- # Calculate column widths for consistent formatting+ # Calculate column widths for consistent formatting (add details column)column_widths = calculate_column_widths(all_provider_data, feature_mapping)+ column_widths['details'] = len('Details')# Sort providers by support score (descending) then by name (ascending)sorted_providers = sorted(@@ -364,7 +365,7 @@ def process_model_sweep_report(input_file, output_file, config_file=None, debug=key=lambda x: (-provider_support_scores[x], x))- # Generate output for each provider+ # Generate tables for all providers firstfor provider_name in sorted_providers:model_data = all_provider_data[provider_name]support_score = provider_support_scores[provider_name]@@ -378,7 +379,8 @@ def process_model_sweep_report(input_file, output_file, config_file=None, debug=header_parts.append(f"{feature:^{column_widths[feature]}}")header_parts.extend([f"{'Context Window':^{column_widths['context_window']}}",- f"{'Last Scanned':^{column_widths['last_scanned']}}"+ f"{'Last Scanned':^{column_widths['last_scanned']}}",+ f"{'Details':^{column_widths['details']}}"])header_row = "| " + " | ".join(header_parts) + " |"@@ -388,7 +390,8 @@ def process_model_sweep_report(input_file, output_file, config_file=None, debug=separator_parts.append(f":{'-' * (column_widths[feature] - 2)}:")separator_parts.extend([f":{'-' * (column_widths['context_window'] - 2)}:",- f":{'-' * (column_widths['last_scanned'] - 2)}:"+ f":{'-' * (column_widths['last_scanned'] - 2)}:",+ f":{'-' * (column_widths['details'] - 2)}:"])separator_row = "|" + "|".join(separator_parts) + "|"@@ -404,6 +407,7 @@ def process_model_sweep_report(input_file, output_file, config_file=None, debug=for model_info in model_data:# Create anchor for model detailsmodel_anchor = model_info['name'].replace('/', '_').replace(':', '_').replace('-', '_').lower()+ details_anchor = f"{provider_name.lower().replace(' ', '_')}_{model_anchor}_details"# Build row with left-aligned first column, centered othersrow_parts = [f"`{model_info['name']}`".ljust(column_widths['model'])]@@ -411,25 +415,35 @@ def process_model_sweep_report(input_file, output_file, config_file=None, debug=row_parts.append(f"{model_info['feature_support'][feature]:^{column_widths[feature]}}")row_parts.extend([f"{model_info['context_window']:,}".center(column_widths['context_window']),- f"{model_info['last_scanned']}".center(column_widths['last_scanned'])+ f"{model_info['last_scanned']}".center(column_widths['last_scanned']),+ f"[View](#{details_anchor})".center(column_widths['details'])])row = "| " + " | ".join(row_parts) + " |"mdx_lines.append(row)- # Add detailed test results for each model- mdx_lines.extend(["", "### Detailed Test Results", ""])+ # Add spacing between provider tables+ mdx_lines.extend(["", ""])++ # Add detailed test results section after all tables+ mdx_lines.extend(["---", "", "# Detailed Test Results", ""])++ for provider_name in sorted_providers:+ model_data = all_provider_data[provider_name]+ mdx_lines.extend([f"## {provider_name}", ""])for model_info in model_data:model_anchor = model_info['name'].replace('/', '_').replace(':', '_').replace('-', '_').lower()- mdx_lines.append(f"#### {model_info['name']}")+ details_anchor = f"{provider_name.lower().replace(' ', '_')}_{model_anchor}_details"+ mdx_lines.append(f"<a id=\"{details_anchor}\"></a>")+ mdx_lines.append(f"### {model_info['name']}")mdx_lines.append("")# Add test detailstest_details = generate_test_details(model_info, feature_mapping)mdx_lines.extend(test_details)- # Add spacing between providers- mdx_lines.extend(["", "---", ""])+ # Add spacing between providers in details section+ mdx_lines.extend(["", ""])# Write the MDX filewith open(output_file, 'w') as f:diff --git a/.github/scripts/model-sweep/model_sweep_report.mdx b/.github/scripts/model-sweep/model_sweep_report.mdxindex 8990e4407..7d43ed2a4 100644--- a/.github/scripts/model-sweep/model_sweep_report.mdx+++ b/.github/scripts/model-sweep/model_sweep_report.mdx@@ -1,6 +1,6 @@---title: Support Models-generated: 2025-06-20T16:24:07.581430+generated: 2025-06-20T16:32:55.429847---# Supported Models@@ -14,21 +14,208 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025## anthropic-| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned |-|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|-| `claude-3-5-haiku-20241022` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 |-| `claude-3-5-sonnet-20241022` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 |-| `claude-3-7-sonnet-20250219` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 |-| `claude-sonnet-4-20250514` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 |-| `claude-opus-4-20250514` | ✅ | ✅ | ⚠️ | 200,000 | 2025-06-20 |-| `claude-3-5-sonnet-20240620` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 |-| `claude-3-haiku-20240307` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 |-| `claude-3-opus-20240229` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 |-| `claude-3-sonnet-20240229` | ❌ | ❌ | ❌ | 200,000 | 2025-06-20 |+| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|+| `claude-3-5-haiku-20241022` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_5_haiku_20241022_details) |+| `claude-3-5-sonnet-20241022` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_5_sonnet_20241022_details) |+| `claude-3-7-sonnet-20250219` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_7_sonnet_20250219_details) |+| `claude-sonnet-4-20250514` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_sonnet_4_20250514_details) |+| `claude-opus-4-20250514` | ✅ | ✅ | ⚠️ | 200,000 | 2025-06-20 | [View](#anthropic_claude_opus_4_20250514_details) |+| `claude-3-5-sonnet-20240620` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_5_sonnet_20240620_details) |+| `claude-3-haiku-20240307` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_haiku_20240307_details) |+| `claude-3-opus-20240229` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_opus_20240229_details) |+| `claude-3-sonnet-20240229` | ❌ | ❌ | ❌ | 200,000 | 2025-06-20 | [View](#anthropic_claude_3_sonnet_20240229_details) |-### Detailed Test Results-#### claude-3-5-haiku-20241022+## openai++| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|+| `gpt-4.1` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_details) |+| `gpt-4.1-2025-04-14` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_2025_04_14_details) |+| `gpt-4.1-nano-2025-04-14` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_nano_2025_04_14_details) |+| `gpt-4o` | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_details) |+| `gpt-4o-2024-05-13` | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_2024_05_13_details) |+| `gpt-4-turbo` | ✅ | ✅ | ⚠️ | 8,192 | 2025-06-20 | [View](#openai_gpt_4_turbo_details) |+| `gpt-4.1-mini` | ✅ | ✅ | ⚠️ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_mini_details) |+| `gpt-4.5-preview` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | [View](#openai_gpt_4.5_preview_details) |+| `gpt-4.5-preview-2025-02-27` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | [View](#openai_gpt_4.5_preview_2025_02_27_details) |+| `gpt-4o-2024-08-06` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_2024_08_06_details) |+| `gpt-4-0613` | ✅ | ✅ | ❌ | 8,192 | 2025-06-20 | [View](#openai_gpt_4_0613_details) |+| `gpt-4-1106-preview` | ✅ | ✅ | ❌ | 128,000 | 2025-06-20 | [View](#openai_gpt_4_1106_preview_details) |+| `gpt-4-turbo-2024-04-09` | ✅ | ⚠️ | ✅ | 128,000 | 2025-06-20 | [View](#openai_gpt_4_turbo_2024_04_09_details) |+| `gpt-4.1-mini-2025-04-14` | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_mini_2025_04_14_details) |+| `gpt-4.1-nano` | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | [View](#openai_gpt_4.1_nano_details) |+| `gpt-4o-2024-11-20` | ⚠️ | ✅ | ✅ | 8,192 | 2025-06-20 | [View](#openai_gpt_4o_2024_11_20_details) |+| `gpt-4-turbo-preview` | ✅ | ⚠️ | ❌ | 128,000 | 2025-06-20 | [View](#openai_gpt_4_turbo_preview_details) |+| `gpt-4-0125-preview` | ⚠️ | ✅ | ❌ | 128,000 | 2025-06-20 | [View](#openai_gpt_4_0125_preview_details) |+| `gpt-4o-mini` | ⚠️ | ⚠️ | ⚠️ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_mini_details) |+| `gpt-4o-mini-2024-07-18` | ⚠️ | ⚠️ | ❌ | 128,000 | 2025-06-20 | [View](#openai_gpt_4o_mini_2024_07_18_details) |+| `gpt-4` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_gpt_4_details) |+| `o1` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o1_details) |+| `o1-2024-12-17` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o1_2024_12_17_details) |+| `o3` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_details) |+| `o3-2025-04-16` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_2025_04_16_details) |+| `o3-mini` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_mini_details) |+| `o3-mini-2025-01-31` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_mini_2025_01_31_details) |+| `o3-pro` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_pro_details) |+| `o3-pro-2025-06-10` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#openai_o3_pro_2025_06_10_details) |+++## google_ai++| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|+| `gemini-1.5-pro` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_pro_details) |+| `gemini-1.5-pro-002` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_pro_002_details) |+| `gemini-1.5-pro-latest` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_pro_latest_details) |+| `gemini-2.5-flash-preview-04-17-thinking` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_flash_preview_04_17_thinking_details) |+| `gemini-2.5-pro-preview-03-25` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_preview_03_25_details) |+| `gemini-2.5-pro-preview-05-06` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_preview_05_06_details) |+| `gemini-2.5-flash-preview-05-20` | ✅ | ⚠️ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_flash_preview_05_20_details) |+| `gemini-2.0-flash-thinking-exp` | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_thinking_exp_details) |+| `gemini-2.0-flash-thinking-exp-1219` | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_thinking_exp_1219_details) |+| `gemini-2.0-flash-thinking-exp-01-21` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_thinking_exp_01_21_details) |+| `gemini-2.5-flash-preview-04-17` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_flash_preview_04_17_details) |+| `gemini-2.5-pro-preview-06-05` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_preview_06_05_details) |+| `gemini-1.0-pro-vision-latest` | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | [View](#google_ai_gemini_1.0_pro_vision_latest_details) |+| `gemini-1.5-flash` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_details) |+| `gemini-1.5-flash-002` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_002_details) |+| `gemini-1.5-flash-8b` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_8b_details) |+| `gemini-1.5-flash-8b-001` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_8b_001_details) |+| `gemini-1.5-flash-8b-latest` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_8b_latest_details) |+| `gemini-1.5-flash-latest` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | [View](#google_ai_gemini_1.5_flash_latest_details) |+| `gemini-2.0-flash` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_details) |+| `gemini-2.0-flash-001` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_001_details) |+| `gemini-2.0-flash-exp` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_exp_details) |+| `gemini-2.0-flash-exp-image-generation` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_exp_image_generation_details) |+| `gemini-2.0-flash-lite` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_lite_details) |+| `gemini-2.0-flash-lite-001` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_lite_001_details) |+| `gemini-2.0-flash-lite-preview` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_lite_preview_details) |+| `gemini-2.0-flash-lite-preview-02-05` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_lite_preview_02_05_details) |+| `gemini-2.0-flash-preview-image-generation` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#google_ai_gemini_2.0_flash_preview_image_generation_details) |+| `gemini-2.0-pro-exp` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_pro_exp_details) |+| `gemini-2.0-pro-exp-02-05` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.0_pro_exp_02_05_details) |+| `gemini-2.5-flash-preview-tts` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#google_ai_gemini_2.5_flash_preview_tts_details) |+| `gemini-2.5-pro-exp-03-25` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_exp_03_25_details) |+| `gemini-2.5-pro-preview-tts` | ❌ | ❌ | ❌ | 65,536 | 2025-06-20 | [View](#google_ai_gemini_2.5_pro_preview_tts_details) |+| `gemini-exp-1206` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | [View](#google_ai_gemini_exp_1206_details) |+| `gemini-pro-vision` | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | [View](#google_ai_gemini_pro_vision_details) |+++## letta++| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|+| `letta-free` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#letta_letta_free_details) |+++## together++| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|+| `Qwen/Qwen2.5-72B-Instruct-Turbo` | ✅ | ✅ | ⚠️ | 131,072 | 2025-06-20 | [View](#together_qwen_qwen2.5_72b_instruct_turbo_details) |+| `arcee-ai/virtuoso-large` | ⚠️ | ✅ | ✅ | 131,072 | 2025-06-20 | [View](#together_arcee_ai_virtuoso_large_details) |+| `Qwen/QwQ-32B` | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | [View](#together_qwen_qwq_32b_details) |+| `Qwen/Qwen2.5-7B-Instruct-Turbo` | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | [View](#together_qwen_qwen2.5_7b_instruct_turbo_details) |+| `Qwen/Qwen2.5-Coder-32B-Instruct` | ⚠️ | ✅ | ⚠️ | 16,384 | 2025-06-20 | [View](#together_qwen_qwen2.5_coder_32b_instruct_details) |+| `arcee-ai/coder-large` | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | [View](#together_arcee_ai_coder_large_details) |+| `arcee_ai/arcee-spotlight` | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | [View](#together_arcee_ai_arcee_spotlight_details) |+| `meta-llama/Llama-3.2-3B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.2_3b_instruct_turbo_details) |+| `meta-llama/Llama-3.3-70B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.3_70b_instruct_turbo_details) |+| `meta-llama/Llama-3.3-70B-Instruct-Turbo-Free` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.3_70b_instruct_turbo_free_details) |+| `meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 130,815 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details) |+| `meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details) |+| `nvidia/Llama-3.1-Nemotron-70B-Instruct-HF` | ⚠️ | ✅ | ❌ | 32,768 | 2025-06-20 | [View](#together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details) |+| `arcee-ai/virtuoso-medium-v2` | ⚠️ | ⚠️ | ✅ | 131,072 | 2025-06-20 | [View](#together_arcee_ai_virtuoso_medium_v2_details) |+| `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8` | ⚠️ | ❌ | ✅ | 1,048,576 | 2025-06-20 | [View](#together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details) |+| `Qwen/Qwen3-235B-A22B-fp8-tput` | ⚠️ | ⚠️ | ❌ | 40,960 | 2025-06-20 | [View](#together_qwen_qwen3_235b_a22b_fp8_tput_details) |+| `deepseek-ai/DeepSeek-V3` | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_v3_details) |+| `meta-llama/Llama-4-Scout-17B-16E-Instruct` | ⚠️ | ⚠️ | ❌ | 1,048,576 | 2025-06-20 | [View](#together_meta_llama_llama_4_scout_17b_16e_instruct_details) |+| `meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo` | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details) |+| `mistralai/Mixtral-8x7B-Instruct-v0.1` | ⚠️ | ⚠️ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mixtral_8x7b_instruct_v0.1_details) |+| `arcee-ai/caller` | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | [View](#together_arcee_ai_caller_details) |+| `mistralai/Mistral-Small-24B-Instruct-2501` | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mistral_small_24b_instruct_2501_details) |+| `NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details) |+| `Qwen/Qwen2-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_qwen_qwen2_72b_instruct_details) |+| `Qwen/Qwen2-VL-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_qwen_qwen2_vl_72b_instruct_details) |+| `Qwen/Qwen2.5-VL-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_qwen_qwen2.5_vl_72b_instruct_details) |+| `arcee-ai/arcee-blitz` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_arcee_ai_arcee_blitz_details) |+| `arcee-ai/maestro-reasoning` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_arcee_ai_maestro_reasoning_details) |+| `deepseek-ai/DeepSeek-R1` | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_details) |+| `deepseek-ai/DeepSeek-R1-Distill-Llama-70B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_distill_llama_70b_details) |+| `deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details) |+| `deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details) |+| `deepseek-ai/DeepSeek-R1-Distill-Qwen-14B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_r1_distill_qwen_14b_details) |+| `deepseek-ai/DeepSeek-V3-p-dp` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_deepseek_ai_deepseek_v3_p_dp_details) |+| `google/gemma-2-27b-it` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_google_gemma_2_27b_it_details) |+| `lgai/exaone-3-5-32b-instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_lgai_exaone_3_5_32b_instruct_details) |+| `lgai/exaone-deep-32b` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_lgai_exaone_deep_32b_details) |+| `marin-community/marin-8b-instruct` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_marin_community_marin_8b_instruct_details) |+| `meta-llama/Llama-3-70b-chat-hf` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_meta_llama_llama_3_70b_chat_hf_details) |+| `meta-llama/Llama-3-8b-chat-hf` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_meta_llama_llama_3_8b_chat_hf_details) |+| `meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details) |+| `meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details) |+| `meta-llama/Llama-Vision-Free` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | [View](#together_meta_llama_llama_vision_free_details) |+| `meta-llama/Meta-Llama-3-70B-Instruct-Turbo` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3_70b_instruct_turbo_details) |+| `meta-llama/Meta-Llama-3-8B-Instruct-Lite` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_meta_llama_meta_llama_3_8b_instruct_lite_details) |+| `mistralai/Mistral-7B-Instruct-v0.1` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mistral_7b_instruct_v0.1_details) |+| `mistralai/Mistral-7B-Instruct-v0.2` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mistral_7b_instruct_v0.2_details) |+| `mistralai/Mistral-7B-Instruct-v0.3` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_mistralai_mistral_7b_instruct_v0.3_details) |+| `perplexity-ai/r1-1776` | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | [View](#together_perplexity_ai_r1_1776_details) |+| `scb10x/scb10x-llama3-1-typhoon2-70b-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details) |+| `scb10x/scb10x-typhoon-2-1-gemma3-12b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details) |+| `togethercomputer/MoA-1` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_togethercomputer_moa_1_details) |+| `togethercomputer/MoA-1-Turbo` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | [View](#together_togethercomputer_moa_1_turbo_details) |+| `togethercomputer/Refuel-Llm-V2` | ❌ | ❌ | ❌ | 16,384 | 2025-06-20 | [View](#together_togethercomputer_refuel_llm_v2_details) |+| `togethercomputer/Refuel-Llm-V2-Small` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#together_togethercomputer_refuel_llm_v2_small_details) |+++## deepseek++| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|+| `deepseek-chat` | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | [View](#deepseek_deepseek_chat_details) |+| `deepseek-reasoner` | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | [View](#deepseek_deepseek_reasoner_details) |+++## groq++| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | Details |+|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|:-----:|+| `allam-2-7b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_allam_2_7b_details) |+| `compound-beta` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_compound_beta_details) |+| `compound-beta-mini` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_compound_beta_mini_details) |+| `deepseek-r1-distill-llama-70b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_deepseek_r1_distill_llama_70b_details) |+| `distil-whisper-large-v3-en` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_distil_whisper_large_v3_en_details) |+| `gemma2-9b-it` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_gemma2_9b_it_details) |+| `llama-3.1-8b-instant` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama_3.1_8b_instant_details) |+| `llama-3.3-70b-versatile` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama_3.3_70b_versatile_details) |+| `llama-guard-3-8b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama_guard_3_8b_details) |+| `llama3-70b-8192` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama3_70b_8192_details) |+| `llama3-8b-8192` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_llama3_8b_8192_details) |+| `meta-llama/llama-4-maverick-17b-128e-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_4_maverick_17b_128e_instruct_details) |+| `meta-llama/llama-4-scout-17b-16e-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_4_scout_17b_16e_instruct_details) |+| `meta-llama/llama-guard-4-12b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_guard_4_12b_details) |+| `meta-llama/llama-prompt-guard-2-22m` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_prompt_guard_2_22m_details) |+| `meta-llama/llama-prompt-guard-2-86m` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_meta_llama_llama_prompt_guard_2_86m_details) |+| `mistral-saba-24b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_mistral_saba_24b_details) |+| `playai-tts` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_playai_tts_details) |+| `playai-tts-arabic` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_playai_tts_arabic_details) |+| `qwen-qwq-32b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_qwen_qwq_32b_details) |+| `qwen/qwen3-32b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_qwen_qwen3_32b_details) |+| `whisper-large-v3` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_whisper_large_v3_details) |+| `whisper-large-v3-turbo` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | [View](#groq_whisper_large_v3_turbo_details) |+++---++# Detailed Test Results++## anthropic++<a id="anthropic_claude_3_5_haiku_20241022_details"></a>+### claude-3-5-haiku-20241022### Basic@@ -55,7 +242,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### claude-3-5-sonnet-20241022+<a id="anthropic_claude_3_5_sonnet_20241022_details"></a>+### claude-3-5-sonnet-20241022### Basic@@ -82,7 +270,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### claude-3-7-sonnet-20250219+<a id="anthropic_claude_3_7_sonnet_20250219_details"></a>+### claude-3-7-sonnet-20250219### Basic@@ -109,7 +298,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### claude-sonnet-4-20250514+<a id="anthropic_claude_sonnet_4_20250514_details"></a>+### claude-sonnet-4-20250514### Basic@@ -136,7 +326,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### claude-opus-4-20250514+<a id="anthropic_claude_opus_4_20250514_details"></a>+### claude-opus-4-20250514### Basic@@ -163,7 +354,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### claude-3-5-sonnet-20240620+<a id="anthropic_claude_3_5_sonnet_20240620_details"></a>+### claude-3-5-sonnet-20240620### Basic@@ -190,7 +382,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### claude-3-haiku-20240307+<a id="anthropic_claude_3_haiku_20240307_details"></a>+### claude-3-haiku-20240307### Basic@@ -217,7 +410,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### claude-3-opus-20240229+<a id="anthropic_claude_3_opus_20240229_details"></a>+### claude-3-opus-20240229### Basic@@ -244,7 +438,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### claude-3-sonnet-20240229+<a id="anthropic_claude_3_sonnet_20240229_details"></a>+### claude-3-sonnet-20240229### Basic@@ -272,45 +467,11 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)----## openai-| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned |-|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|-| `gpt-4.1` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 |-| `gpt-4.1-2025-04-14` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 |-| `gpt-4.1-nano-2025-04-14` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 |-| `gpt-4o` | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 |-| `gpt-4o-2024-05-13` | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 |-| `gpt-4-turbo` | ✅ | ✅ | ⚠️ | 8,192 | 2025-06-20 |-| `gpt-4.1-mini` | ✅ | ✅ | ⚠️ | 1,047,576 | 2025-06-20 |-| `gpt-4.5-preview` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 |-| `gpt-4.5-preview-2025-02-27` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 |-| `gpt-4o-2024-08-06` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 |-| `gpt-4-0613` | ✅ | ✅ | ❌ | 8,192 | 2025-06-20 |-| `gpt-4-1106-preview` | ✅ | ✅ | ❌ | 128,000 | 2025-06-20 |-| `gpt-4-turbo-2024-04-09` | ✅ | ⚠️ | ✅ | 128,000 | 2025-06-20 |-| `gpt-4.1-mini-2025-04-14` | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 |-| `gpt-4.1-nano` | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 |-| `gpt-4o-2024-11-20` | ⚠️ | ✅ | ✅ | 8,192 | 2025-06-20 |-| `gpt-4-turbo-preview` | ✅ | ⚠️ | ❌ | 128,000 | 2025-06-20 |-| `gpt-4-0125-preview` | ⚠️ | ✅ | ❌ | 128,000 | 2025-06-20 |-| `gpt-4o-mini` | ⚠️ | ⚠️ | ⚠️ | 128,000 | 2025-06-20 |-| `gpt-4o-mini-2024-07-18` | ⚠️ | ⚠️ | ❌ | 128,000 | 2025-06-20 |-| `gpt-4` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `o1` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `o1-2024-12-17` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `o3` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `o3-2025-04-16` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `o3-mini` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `o3-mini-2025-01-31` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `o3-pro` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `o3-pro-2025-06-10` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |--### Detailed Test Results--#### gpt-4.1+<a id="openai_gpt_4.1_details"></a>+### gpt-4.1### Basic@@ -337,7 +498,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4.1-2025-04-14+<a id="openai_gpt_4.1_2025_04_14_details"></a>+### gpt-4.1-2025-04-14### Basic@@ -364,7 +526,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4.1-nano-2025-04-14+<a id="openai_gpt_4.1_nano_2025_04_14_details"></a>+### gpt-4.1-nano-2025-04-14### Basic@@ -391,7 +554,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4o+<a id="openai_gpt_4o_details"></a>+### gpt-4o### Basic@@ -418,7 +582,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4o-2024-05-13+<a id="openai_gpt_4o_2024_05_13_details"></a>+### gpt-4o-2024-05-13### Basic@@ -445,7 +610,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4-turbo+<a id="openai_gpt_4_turbo_details"></a>+### gpt-4-turbo### Basic@@ -472,7 +638,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4.1-mini+<a id="openai_gpt_4.1_mini_details"></a>+### gpt-4.1-mini### Basic@@ -499,7 +666,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4.5-preview+<a id="openai_gpt_4.5_preview_details"></a>+### gpt-4.5-preview### Basic@@ -526,7 +694,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4.5-preview-2025-02-27+<a id="openai_gpt_4.5_preview_2025_02_27_details"></a>+### gpt-4.5-preview-2025-02-27### Basic@@ -553,7 +722,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4o-2024-08-06+<a id="openai_gpt_4o_2024_08_06_details"></a>+### gpt-4o-2024-08-06### Basic@@ -580,7 +750,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4-0613+<a id="openai_gpt_4_0613_details"></a>+### gpt-4-0613### Basic@@ -607,7 +778,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4-1106-preview+<a id="openai_gpt_4_1106_preview_details"></a>+### gpt-4-1106-preview### Basic@@ -634,7 +806,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4-turbo-2024-04-09+<a id="openai_gpt_4_turbo_2024_04_09_details"></a>+### gpt-4-turbo-2024-04-09### Basic@@ -661,7 +834,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4.1-mini-2025-04-14+<a id="openai_gpt_4.1_mini_2025_04_14_details"></a>+### gpt-4.1-mini-2025-04-14### Basic@@ -688,7 +862,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4.1-nano+<a id="openai_gpt_4.1_nano_details"></a>+### gpt-4.1-nano### Basic@@ -715,7 +890,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4o-2024-11-20+<a id="openai_gpt_4o_2024_11_20_details"></a>+### gpt-4o-2024-11-20### Basic@@ -742,7 +918,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4-turbo-preview+<a id="openai_gpt_4_turbo_preview_details"></a>+### gpt-4-turbo-preview### Basic@@ -769,7 +946,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4-0125-preview+<a id="openai_gpt_4_0125_preview_details"></a>+### gpt-4-0125-preview### Basic@@ -796,7 +974,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4o-mini+<a id="openai_gpt_4o_mini_details"></a>+### gpt-4o-mini### Basic@@ -823,7 +1002,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4o-mini-2024-07-18+<a id="openai_gpt_4o_mini_2024_07_18_details"></a>+### gpt-4o-mini-2024-07-18### Basic@@ -850,7 +1030,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gpt-4+<a id="openai_gpt_4_details"></a>+### gpt-4### Basic@@ -877,7 +1058,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### o1+<a id="openai_o1_details"></a>+### o1### Basic@@ -904,7 +1086,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### o1-2024-12-17+<a id="openai_o1_2024_12_17_details"></a>+### o1-2024-12-17### Basic@@ -931,7 +1114,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### o3+<a id="openai_o3_details"></a>+### o3### Basic@@ -958,7 +1142,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### o3-2025-04-16+<a id="openai_o3_2025_04_16_details"></a>+### o3-2025-04-16### Basic@@ -985,7 +1170,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### o3-mini+<a id="openai_o3_mini_details"></a>+### o3-mini### Basic@@ -1012,7 +1198,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### o3-mini-2025-01-31+<a id="openai_o3_mini_2025_01_31_details"></a>+### o3-mini-2025-01-31### Basic@@ -1039,7 +1226,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### o3-pro+<a id="openai_o3_pro_details"></a>+### o3-pro### Basic@@ -1066,7 +1254,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### o3-pro-2025-06-10+<a id="openai_o3_pro_2025_06_10_details"></a>+### o3-pro-2025-06-10### Basic@@ -1094,51 +1283,11 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)----## google_ai-| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned |-|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|-| `gemini-1.5-pro` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 |-| `gemini-1.5-pro-002` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 |-| `gemini-1.5-pro-latest` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 |-| `gemini-2.5-flash-preview-04-17-thinking` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 |-| `gemini-2.5-pro-preview-03-25` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 |-| `gemini-2.5-pro-preview-05-06` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 |-| `gemini-2.5-flash-preview-05-20` | ✅ | ⚠️ | ✅ | 1,048,576 | 2025-06-20 |-| `gemini-2.0-flash-thinking-exp` | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 |-| `gemini-2.0-flash-thinking-exp-1219` | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 |-| `gemini-2.0-flash-thinking-exp-01-21` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 |-| `gemini-2.5-flash-preview-04-17` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 |-| `gemini-2.5-pro-preview-06-05` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 |-| `gemini-1.0-pro-vision-latest` | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 |-| `gemini-1.5-flash` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 |-| `gemini-1.5-flash-002` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 |-| `gemini-1.5-flash-8b` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 |-| `gemini-1.5-flash-8b-001` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 |-| `gemini-1.5-flash-8b-latest` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 |-| `gemini-1.5-flash-latest` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 |-| `gemini-2.0-flash` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |-| `gemini-2.0-flash-001` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |-| `gemini-2.0-flash-exp` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |-| `gemini-2.0-flash-exp-image-generation` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |-| `gemini-2.0-flash-lite` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |-| `gemini-2.0-flash-lite-001` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |-| `gemini-2.0-flash-lite-preview` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |-| `gemini-2.0-flash-lite-preview-02-05` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |-| `gemini-2.0-flash-preview-image-generation` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |-| `gemini-2.0-pro-exp` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |-| `gemini-2.0-pro-exp-02-05` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |-| `gemini-2.5-flash-preview-tts` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |-| `gemini-2.5-pro-exp-03-25` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |-| `gemini-2.5-pro-preview-tts` | ❌ | ❌ | ❌ | 65,536 | 2025-06-20 |-| `gemini-exp-1206` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 |-| `gemini-pro-vision` | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 |--### Detailed Test Results--#### gemini-1.5-pro+<a id="google_ai_gemini_1.5_pro_details"></a>+### gemini-1.5-pro### Basic@@ -1165,7 +1314,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-1.5-pro-002+<a id="google_ai_gemini_1.5_pro_002_details"></a>+### gemini-1.5-pro-002### Basic@@ -1192,7 +1342,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-1.5-pro-latest+<a id="google_ai_gemini_1.5_pro_latest_details"></a>+### gemini-1.5-pro-latest### Basic@@ -1219,7 +1370,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.5-flash-preview-04-17-thinking+<a id="google_ai_gemini_2.5_flash_preview_04_17_thinking_details"></a>+### gemini-2.5-flash-preview-04-17-thinking### Basic@@ -1246,7 +1398,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.5-pro-preview-03-25+<a id="google_ai_gemini_2.5_pro_preview_03_25_details"></a>+### gemini-2.5-pro-preview-03-25### Basic@@ -1273,7 +1426,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.5-pro-preview-05-06+<a id="google_ai_gemini_2.5_pro_preview_05_06_details"></a>+### gemini-2.5-pro-preview-05-06### Basic@@ -1300,7 +1454,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.5-flash-preview-05-20+<a id="google_ai_gemini_2.5_flash_preview_05_20_details"></a>+### gemini-2.5-flash-preview-05-20### Basic@@ -1327,7 +1482,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.0-flash-thinking-exp+<a id="google_ai_gemini_2.0_flash_thinking_exp_details"></a>+### gemini-2.0-flash-thinking-exp### Basic@@ -1354,7 +1510,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.0-flash-thinking-exp-1219+<a id="google_ai_gemini_2.0_flash_thinking_exp_1219_details"></a>+### gemini-2.0-flash-thinking-exp-1219### Basic@@ -1381,7 +1538,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.0-flash-thinking-exp-01-21+<a id="google_ai_gemini_2.0_flash_thinking_exp_01_21_details"></a>+### gemini-2.0-flash-thinking-exp-01-21### Basic@@ -1408,7 +1566,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.5-flash-preview-04-17+<a id="google_ai_gemini_2.5_flash_preview_04_17_details"></a>+### gemini-2.5-flash-preview-04-17### Basic@@ -1435,7 +1594,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.5-pro-preview-06-05+<a id="google_ai_gemini_2.5_pro_preview_06_05_details"></a>+### gemini-2.5-pro-preview-06-05### Basic@@ -1462,7 +1622,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-1.0-pro-vision-latest+<a id="google_ai_gemini_1.0_pro_vision_latest_details"></a>+### gemini-1.0-pro-vision-latest### Basic@@ -1489,7 +1650,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-1.5-flash+<a id="google_ai_gemini_1.5_flash_details"></a>+### gemini-1.5-flash### Basic@@ -1516,7 +1678,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-1.5-flash-002+<a id="google_ai_gemini_1.5_flash_002_details"></a>+### gemini-1.5-flash-002### Basic@@ -1543,7 +1706,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-1.5-flash-8b+<a id="google_ai_gemini_1.5_flash_8b_details"></a>+### gemini-1.5-flash-8b### Basic@@ -1570,7 +1734,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-1.5-flash-8b-001+<a id="google_ai_gemini_1.5_flash_8b_001_details"></a>+### gemini-1.5-flash-8b-001### Basic@@ -1597,7 +1762,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-1.5-flash-8b-latest+<a id="google_ai_gemini_1.5_flash_8b_latest_details"></a>+### gemini-1.5-flash-8b-latest### Basic@@ -1624,7 +1790,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-1.5-flash-latest+<a id="google_ai_gemini_1.5_flash_latest_details"></a>+### gemini-1.5-flash-latest### Basic@@ -1651,7 +1818,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.0-flash+<a id="google_ai_gemini_2.0_flash_details"></a>+### gemini-2.0-flash### Basic@@ -1678,7 +1846,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.0-flash-001+<a id="google_ai_gemini_2.0_flash_001_details"></a>+### gemini-2.0-flash-001### Basic@@ -1705,7 +1874,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.0-flash-exp+<a id="google_ai_gemini_2.0_flash_exp_details"></a>+### gemini-2.0-flash-exp### Basic@@ -1732,7 +1902,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.0-flash-exp-image-generation+<a id="google_ai_gemini_2.0_flash_exp_image_generation_details"></a>+### gemini-2.0-flash-exp-image-generation### Basic@@ -1759,7 +1930,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.0-flash-lite+<a id="google_ai_gemini_2.0_flash_lite_details"></a>+### gemini-2.0-flash-lite### Basic@@ -1786,7 +1958,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.0-flash-lite-001+<a id="google_ai_gemini_2.0_flash_lite_001_details"></a>+### gemini-2.0-flash-lite-001### Basic@@ -1813,7 +1986,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.0-flash-lite-preview+<a id="google_ai_gemini_2.0_flash_lite_preview_details"></a>+### gemini-2.0-flash-lite-preview### Basic@@ -1840,7 +2014,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.0-flash-lite-preview-02-05+<a id="google_ai_gemini_2.0_flash_lite_preview_02_05_details"></a>+### gemini-2.0-flash-lite-preview-02-05### Basic@@ -1867,7 +2042,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.0-flash-preview-image-generation+<a id="google_ai_gemini_2.0_flash_preview_image_generation_details"></a>+### gemini-2.0-flash-preview-image-generation### Basic@@ -1894,7 +2070,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.0-pro-exp+<a id="google_ai_gemini_2.0_pro_exp_details"></a>+### gemini-2.0-pro-exp### Basic@@ -1921,7 +2098,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.0-pro-exp-02-05+<a id="google_ai_gemini_2.0_pro_exp_02_05_details"></a>+### gemini-2.0-pro-exp-02-05### Basic@@ -1948,7 +2126,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.5-flash-preview-tts+<a id="google_ai_gemini_2.5_flash_preview_tts_details"></a>+### gemini-2.5-flash-preview-tts### Basic@@ -1975,7 +2154,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.5-pro-exp-03-25+<a id="google_ai_gemini_2.5_pro_exp_03_25_details"></a>+### gemini-2.5-pro-exp-03-25### Basic@@ -2002,7 +2182,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-2.5-pro-preview-tts+<a id="google_ai_gemini_2.5_pro_preview_tts_details"></a>+### gemini-2.5-pro-preview-tts### Basic@@ -2029,7 +2210,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-exp-1206+<a id="google_ai_gemini_exp_1206_details"></a>+### gemini-exp-1206### Basic@@ -2056,7 +2238,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemini-pro-vision+<a id="google_ai_gemini_pro_vision_details"></a>+### gemini-pro-vision### Basic@@ -2084,17 +2267,11 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)----## letta-| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned |-|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|-| `letta-free` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 |--### Detailed Test Results--#### letta-free+<a id="letta_letta_free_details"></a>+### letta-free### Basic@@ -2122,71 +2299,11 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)----## together-| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned |-|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|-| `Qwen/Qwen2.5-72B-Instruct-Turbo` | ✅ | ✅ | ⚠️ | 131,072 | 2025-06-20 |-| `arcee-ai/virtuoso-large` | ⚠️ | ✅ | ✅ | 131,072 | 2025-06-20 |-| `Qwen/QwQ-32B` | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 |-| `Qwen/Qwen2.5-7B-Instruct-Turbo` | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 |-| `Qwen/Qwen2.5-Coder-32B-Instruct` | ⚠️ | ✅ | ⚠️ | 16,384 | 2025-06-20 |-| `arcee-ai/coder-large` | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 |-| `arcee_ai/arcee-spotlight` | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 |-| `meta-llama/Llama-3.2-3B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 |-| `meta-llama/Llama-3.3-70B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 |-| `meta-llama/Llama-3.3-70B-Instruct-Turbo-Free` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 |-| `meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 130,815 | 2025-06-20 |-| `meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 |-| `nvidia/Llama-3.1-Nemotron-70B-Instruct-HF` | ⚠️ | ✅ | ❌ | 32,768 | 2025-06-20 |-| `arcee-ai/virtuoso-medium-v2` | ⚠️ | ⚠️ | ✅ | 131,072 | 2025-06-20 |-| `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8` | ⚠️ | ❌ | ✅ | 1,048,576 | 2025-06-20 |-| `Qwen/Qwen3-235B-A22B-fp8-tput` | ⚠️ | ⚠️ | ❌ | 40,960 | 2025-06-20 |-| `deepseek-ai/DeepSeek-V3` | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 |-| `meta-llama/Llama-4-Scout-17B-16E-Instruct` | ⚠️ | ⚠️ | ❌ | 1,048,576 | 2025-06-20 |-| `meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo` | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 |-| `mistralai/Mixtral-8x7B-Instruct-v0.1` | ⚠️ | ⚠️ | ❌ | 32,768 | 2025-06-20 |-| `arcee-ai/caller` | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 |-| `mistralai/Mistral-Small-24B-Instruct-2501` | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 |-| `NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |-| `Qwen/Qwen2-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |-| `Qwen/Qwen2-VL-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |-| `Qwen/Qwen2.5-VL-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |-| `arcee-ai/arcee-blitz` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |-| `arcee-ai/maestro-reasoning` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |-| `deepseek-ai/DeepSeek-R1` | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 |-| `deepseek-ai/DeepSeek-R1-Distill-Llama-70B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |-| `deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |-| `deepseek-ai/DeepSeek-R1-Distill-Qwen-14B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |-| `deepseek-ai/DeepSeek-V3-p-dp` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |-| `google/gemma-2-27b-it` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `lgai/exaone-3-5-32b-instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |-| `lgai/exaone-deep-32b` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |-| `marin-community/marin-8b-instruct` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |-| `meta-llama/Llama-3-70b-chat-hf` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `meta-llama/Llama-3-8b-chat-hf` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |-| `meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |-| `meta-llama/Llama-Vision-Free` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 |-| `meta-llama/Meta-Llama-3-70B-Instruct-Turbo` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `meta-llama/Meta-Llama-3-8B-Instruct-Lite` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `mistralai/Mistral-7B-Instruct-v0.1` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |-| `mistralai/Mistral-7B-Instruct-v0.2` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |-| `mistralai/Mistral-7B-Instruct-v0.3` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |-| `perplexity-ai/r1-1776` | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 |-| `scb10x/scb10x-llama3-1-typhoon2-70b-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `scb10x/scb10x-typhoon-2-1-gemma3-12b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `togethercomputer/MoA-1` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |-| `togethercomputer/MoA-1-Turbo` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 |-| `togethercomputer/Refuel-Llm-V2` | ❌ | ❌ | ❌ | 16,384 | 2025-06-20 |-| `togethercomputer/Refuel-Llm-V2-Small` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |--### Detailed Test Results--#### Qwen/Qwen2.5-72B-Instruct-Turbo+<a id="together_qwen_qwen2.5_72b_instruct_turbo_details"></a>+### Qwen/Qwen2.5-72B-Instruct-Turbo### Basic@@ -2213,7 +2330,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### arcee-ai/virtuoso-large+<a id="together_arcee_ai_virtuoso_large_details"></a>+### arcee-ai/virtuoso-large### Basic@@ -2240,7 +2358,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### Qwen/QwQ-32B+<a id="together_qwen_qwq_32b_details"></a>+### Qwen/QwQ-32B### Basic@@ -2267,7 +2386,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### Qwen/Qwen2.5-7B-Instruct-Turbo+<a id="together_qwen_qwen2.5_7b_instruct_turbo_details"></a>+### Qwen/Qwen2.5-7B-Instruct-Turbo### Basic@@ -2294,7 +2414,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### Qwen/Qwen2.5-Coder-32B-Instruct+<a id="together_qwen_qwen2.5_coder_32b_instruct_details"></a>+### Qwen/Qwen2.5-Coder-32B-Instruct### Basic@@ -2321,7 +2442,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### arcee-ai/coder-large+<a id="together_arcee_ai_coder_large_details"></a>+### arcee-ai/coder-large### Basic@@ -2348,7 +2470,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### arcee_ai/arcee-spotlight+<a id="together_arcee_ai_arcee_spotlight_details"></a>+### arcee_ai/arcee-spotlight### Basic@@ -2375,7 +2498,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/Llama-3.2-3B-Instruct-Turbo+<a id="together_meta_llama_llama_3.2_3b_instruct_turbo_details"></a>+### meta-llama/Llama-3.2-3B-Instruct-Turbo### Basic@@ -2402,7 +2526,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/Llama-3.3-70B-Instruct-Turbo+<a id="together_meta_llama_llama_3.3_70b_instruct_turbo_details"></a>+### meta-llama/Llama-3.3-70B-Instruct-Turbo### Basic@@ -2429,7 +2554,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/Llama-3.3-70B-Instruct-Turbo-Free+<a id="together_meta_llama_llama_3.3_70b_instruct_turbo_free_details"></a>+### meta-llama/Llama-3.3-70B-Instruct-Turbo-Free### Basic@@ -2456,7 +2582,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo+<a id="together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details"></a>+### meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo### Basic@@ -2483,7 +2610,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo+<a id="together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details"></a>+### meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo### Basic@@ -2510,7 +2638,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### nvidia/Llama-3.1-Nemotron-70B-Instruct-HF+<a id="together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details"></a>+### nvidia/Llama-3.1-Nemotron-70B-Instruct-HF### Basic@@ -2537,7 +2666,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### arcee-ai/virtuoso-medium-v2+<a id="together_arcee_ai_virtuoso_medium_v2_details"></a>+### arcee-ai/virtuoso-medium-v2### Basic@@ -2564,7 +2694,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8+<a id="together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details"></a>+### meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8### Basic@@ -2591,7 +2722,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### Qwen/Qwen3-235B-A22B-fp8-tput+<a id="together_qwen_qwen3_235b_a22b_fp8_tput_details"></a>+### Qwen/Qwen3-235B-A22B-fp8-tput### Basic@@ -2618,7 +2750,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### deepseek-ai/DeepSeek-V3+<a id="together_deepseek_ai_deepseek_v3_details"></a>+### deepseek-ai/DeepSeek-V3### Basic@@ -2645,7 +2778,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/Llama-4-Scout-17B-16E-Instruct+<a id="together_meta_llama_llama_4_scout_17b_16e_instruct_details"></a>+### meta-llama/Llama-4-Scout-17B-16E-Instruct### Basic@@ -2672,7 +2806,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo+<a id="together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details"></a>+### meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo### Basic@@ -2699,7 +2834,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### mistralai/Mixtral-8x7B-Instruct-v0.1+<a id="together_mistralai_mixtral_8x7b_instruct_v0.1_details"></a>+### mistralai/Mixtral-8x7B-Instruct-v0.1### Basic@@ -2726,7 +2862,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### arcee-ai/caller+<a id="together_arcee_ai_caller_details"></a>+### arcee-ai/caller### Basic@@ -2753,7 +2890,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### mistralai/Mistral-Small-24B-Instruct-2501+<a id="together_mistralai_mistral_small_24b_instruct_2501_details"></a>+### mistralai/Mistral-Small-24B-Instruct-2501### Basic@@ -2780,7 +2918,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO+<a id="together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details"></a>+### NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO### Basic@@ -2807,7 +2946,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### Qwen/Qwen2-72B-Instruct+<a id="together_qwen_qwen2_72b_instruct_details"></a>+### Qwen/Qwen2-72B-Instruct### Basic@@ -2834,7 +2974,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### Qwen/Qwen2-VL-72B-Instruct+<a id="together_qwen_qwen2_vl_72b_instruct_details"></a>+### Qwen/Qwen2-VL-72B-Instruct### Basic@@ -2861,7 +3002,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### Qwen/Qwen2.5-VL-72B-Instruct+<a id="together_qwen_qwen2.5_vl_72b_instruct_details"></a>+### Qwen/Qwen2.5-VL-72B-Instruct### Basic@@ -2888,7 +3030,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### arcee-ai/arcee-blitz+<a id="together_arcee_ai_arcee_blitz_details"></a>+### arcee-ai/arcee-blitz### Basic@@ -2915,7 +3058,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### arcee-ai/maestro-reasoning+<a id="together_arcee_ai_maestro_reasoning_details"></a>+### arcee-ai/maestro-reasoning### Basic@@ -2942,7 +3086,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### deepseek-ai/DeepSeek-R1+<a id="together_deepseek_ai_deepseek_r1_details"></a>+### deepseek-ai/DeepSeek-R1### Basic@@ -2969,7 +3114,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B+<a id="together_deepseek_ai_deepseek_r1_distill_llama_70b_details"></a>+### deepseek-ai/DeepSeek-R1-Distill-Llama-70B### Basic@@ -2996,7 +3142,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free+<a id="together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details"></a>+### deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free### Basic@@ -3023,7 +3170,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B+<a id="together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details"></a>+### deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B### Basic@@ -3050,7 +3198,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### deepseek-ai/DeepSeek-R1-Distill-Qwen-14B+<a id="together_deepseek_ai_deepseek_r1_distill_qwen_14b_details"></a>+### deepseek-ai/DeepSeek-R1-Distill-Qwen-14B### Basic@@ -3077,7 +3226,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### deepseek-ai/DeepSeek-V3-p-dp+<a id="together_deepseek_ai_deepseek_v3_p_dp_details"></a>+### deepseek-ai/DeepSeek-V3-p-dp### Basic@@ -3104,7 +3254,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### google/gemma-2-27b-it+<a id="together_google_gemma_2_27b_it_details"></a>+### google/gemma-2-27b-it### Basic@@ -3131,7 +3282,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### lgai/exaone-3-5-32b-instruct+<a id="together_lgai_exaone_3_5_32b_instruct_details"></a>+### lgai/exaone-3-5-32b-instruct### Basic@@ -3158,7 +3310,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### lgai/exaone-deep-32b+<a id="together_lgai_exaone_deep_32b_details"></a>+### lgai/exaone-deep-32b### Basic@@ -3185,7 +3338,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### marin-community/marin-8b-instruct+<a id="together_marin_community_marin_8b_instruct_details"></a>+### marin-community/marin-8b-instruct### Basic@@ -3212,7 +3366,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/Llama-3-70b-chat-hf+<a id="together_meta_llama_llama_3_70b_chat_hf_details"></a>+### meta-llama/Llama-3-70b-chat-hf### Basic@@ -3239,7 +3394,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/Llama-3-8b-chat-hf+<a id="together_meta_llama_llama_3_8b_chat_hf_details"></a>+### meta-llama/Llama-3-8b-chat-hf### Basic@@ -3266,7 +3422,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo+<a id="together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details"></a>+### meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo### Basic@@ -3293,7 +3450,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo+<a id="together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details"></a>+### meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo### Basic@@ -3320,7 +3478,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/Llama-Vision-Free+<a id="together_meta_llama_llama_vision_free_details"></a>+### meta-llama/Llama-Vision-Free### Basic@@ -3347,7 +3506,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/Meta-Llama-3-70B-Instruct-Turbo+<a id="together_meta_llama_meta_llama_3_70b_instruct_turbo_details"></a>+### meta-llama/Meta-Llama-3-70B-Instruct-Turbo### Basic@@ -3374,7 +3534,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/Meta-Llama-3-8B-Instruct-Lite+<a id="together_meta_llama_meta_llama_3_8b_instruct_lite_details"></a>+### meta-llama/Meta-Llama-3-8B-Instruct-Lite### Basic@@ -3401,7 +3562,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### mistralai/Mistral-7B-Instruct-v0.1+<a id="together_mistralai_mistral_7b_instruct_v0.1_details"></a>+### mistralai/Mistral-7B-Instruct-v0.1### Basic@@ -3428,7 +3590,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### mistralai/Mistral-7B-Instruct-v0.2+<a id="together_mistralai_mistral_7b_instruct_v0.2_details"></a>+### mistralai/Mistral-7B-Instruct-v0.2### Basic@@ -3455,7 +3618,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### mistralai/Mistral-7B-Instruct-v0.3+<a id="together_mistralai_mistral_7b_instruct_v0.3_details"></a>+### mistralai/Mistral-7B-Instruct-v0.3### Basic@@ -3482,7 +3646,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### perplexity-ai/r1-1776+<a id="together_perplexity_ai_r1_1776_details"></a>+### perplexity-ai/r1-1776### Basic@@ -3509,7 +3674,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### scb10x/scb10x-llama3-1-typhoon2-70b-instruct+<a id="together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details"></a>+### scb10x/scb10x-llama3-1-typhoon2-70b-instruct### Basic@@ -3536,7 +3702,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### scb10x/scb10x-typhoon-2-1-gemma3-12b+<a id="together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details"></a>+### scb10x/scb10x-typhoon-2-1-gemma3-12b### Basic@@ -3563,7 +3730,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### togethercomputer/MoA-1+<a id="together_togethercomputer_moa_1_details"></a>+### togethercomputer/MoA-1### Basic@@ -3590,7 +3758,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### togethercomputer/MoA-1-Turbo+<a id="together_togethercomputer_moa_1_turbo_details"></a>+### togethercomputer/MoA-1-Turbo### Basic@@ -3617,7 +3786,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### togethercomputer/Refuel-Llm-V2+<a id="together_togethercomputer_refuel_llm_v2_details"></a>+### togethercomputer/Refuel-Llm-V2### Basic@@ -3644,7 +3814,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### togethercomputer/Refuel-Llm-V2-Small+<a id="together_togethercomputer_refuel_llm_v2_small_details"></a>+### togethercomputer/Refuel-Llm-V2-Small### Basic@@ -3672,18 +3843,11 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)----## deepseek-| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned |-|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|-| `deepseek-chat` | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 |-| `deepseek-reasoner` | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 |--### Detailed Test Results--#### deepseek-chat+<a id="deepseek_deepseek_chat_details"></a>+### deepseek-chat### Basic@@ -3710,7 +3874,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### deepseek-reasoner+<a id="deepseek_deepseek_reasoner_details"></a>+### deepseek-reasoner### Basic@@ -3738,39 +3903,11 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)----## groq-| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned |-|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:|-| `allam-2-7b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `compound-beta` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `compound-beta-mini` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `deepseek-r1-distill-llama-70b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `distil-whisper-large-v3-en` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `gemma2-9b-it` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `llama-3.1-8b-instant` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `llama-3.3-70b-versatile` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `llama-guard-3-8b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `llama3-70b-8192` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `llama3-8b-8192` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `meta-llama/llama-4-maverick-17b-128e-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `meta-llama/llama-4-scout-17b-16e-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `meta-llama/llama-guard-4-12b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `meta-llama/llama-prompt-guard-2-22m` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `meta-llama/llama-prompt-guard-2-86m` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `mistral-saba-24b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `playai-tts` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `playai-tts-arabic` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `qwen-qwq-32b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `qwen/qwen3-32b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `whisper-large-v3` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |-| `whisper-large-v3-turbo` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 |--### Detailed Test Results--#### allam-2-7b+<a id="groq_allam_2_7b_details"></a>+### allam-2-7b### Basic@@ -3797,7 +3934,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### compound-beta+<a id="groq_compound_beta_details"></a>+### compound-beta### Basic@@ -3824,7 +3962,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### compound-beta-mini+<a id="groq_compound_beta_mini_details"></a>+### compound-beta-mini### Basic@@ -3851,7 +3990,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### deepseek-r1-distill-llama-70b+<a id="groq_deepseek_r1_distill_llama_70b_details"></a>+### deepseek-r1-distill-llama-70b### Basic@@ -3878,7 +4018,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### distil-whisper-large-v3-en+<a id="groq_distil_whisper_large_v3_en_details"></a>+### distil-whisper-large-v3-en### Basic@@ -3905,7 +4046,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### gemma2-9b-it+<a id="groq_gemma2_9b_it_details"></a>+### gemma2-9b-it### Basic@@ -3932,7 +4074,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### llama-3.1-8b-instant+<a id="groq_llama_3.1_8b_instant_details"></a>+### llama-3.1-8b-instant### Basic@@ -3959,7 +4102,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### llama-3.3-70b-versatile+<a id="groq_llama_3.3_70b_versatile_details"></a>+### llama-3.3-70b-versatile### Basic@@ -3986,7 +4130,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### llama-guard-3-8b+<a id="groq_llama_guard_3_8b_details"></a>+### llama-guard-3-8b### Basic@@ -4013,7 +4158,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### llama3-70b-8192+<a id="groq_llama3_70b_8192_details"></a>+### llama3-70b-8192### Basic@@ -4040,7 +4186,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### llama3-8b-8192+<a id="groq_llama3_8b_8192_details"></a>+### llama3-8b-8192### Basic@@ -4067,7 +4214,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/llama-4-maverick-17b-128e-instruct+<a id="groq_meta_llama_llama_4_maverick_17b_128e_instruct_details"></a>+### meta-llama/llama-4-maverick-17b-128e-instruct### Basic@@ -4094,7 +4242,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/llama-4-scout-17b-16e-instruct+<a id="groq_meta_llama_llama_4_scout_17b_16e_instruct_details"></a>+### meta-llama/llama-4-scout-17b-16e-instruct### Basic@@ -4121,7 +4270,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/llama-guard-4-12b+<a id="groq_meta_llama_llama_guard_4_12b_details"></a>+### meta-llama/llama-guard-4-12b### Basic@@ -4148,7 +4298,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/llama-prompt-guard-2-22m+<a id="groq_meta_llama_llama_prompt_guard_2_22m_details"></a>+### meta-llama/llama-prompt-guard-2-22m### Basic@@ -4175,7 +4326,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### meta-llama/llama-prompt-guard-2-86m+<a id="groq_meta_llama_llama_prompt_guard_2_86m_details"></a>+### meta-llama/llama-prompt-guard-2-86m### Basic@@ -4202,7 +4354,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### mistral-saba-24b+<a id="groq_mistral_saba_24b_details"></a>+### mistral-saba-24b### Basic@@ -4229,7 +4382,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### playai-tts+<a id="groq_playai_tts_details"></a>+### playai-tts### Basic@@ -4256,7 +4410,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### playai-tts-arabic+<a id="groq_playai_tts_arabic_details"></a>+### playai-tts-arabic### Basic@@ -4283,7 +4438,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### qwen-qwq-32b+<a id="groq_qwen_qwq_32b_details"></a>+### qwen-qwq-32b### Basic@@ -4310,7 +4466,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### qwen/qwen3-32b+<a id="groq_qwen_qwen3_32b_details"></a>+### qwen/qwen3-32b### Basic@@ -4337,7 +4494,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### whisper-large-v3+<a id="groq_whisper_large_v3_details"></a>+### whisper-large-v3### Basic@@ -4364,7 +4522,8 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440)- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)-#### whisper-large-v3-turbo+<a id="groq_whisper_large_v3_turbo_details"></a>+### whisper-large-v3-turbo### Basic@@ -4392,4 +4551,3 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417)----