Personal AI, On Personal Devices
Python
add eval configs (#208)
Avanika Narayan41ddab9099ec
diff --git a/.gitignore b/.gitignoreindex a86fd343..a993fcf3 100644--- a/.gitignore+++ b/.gitignore@@ -90,7 +90,4 @@ CLAUDE.mdresearch_mining_*.python-versionsrc/openjarvis/channels/whatsapp_baileys_bridge/node_modules/--# SQLite in-memory artifacts-:memory:-MagicMock/+scratch/diff --git a/src/openjarvis/evals/configs/gaia-gemini-31-pro.toml b/src/openjarvis/evals/configs/gaia-gemini-31-pro.tomlnew file mode 100644index 00000000..1410012e--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-gemini-31-pro.toml@@ -0,0 +1,29 @@+# GAIA eval: gemini-3.1-pro-preview (cloud)+[meta]+name = "gaia-gemini-31-pro"+description = "GAIA on gemini-3.1-pro-preview"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 1+output_dir = "results/gaia-gemini-31-pro/"+seed = 42++[[models]]+name = "gemini-3.1-pro-preview"+engine = "cloud"++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-gemma4-26b.toml b/src/openjarvis/evals/configs/gaia-gemma4-26b.tomlnew file mode 100644index 00000000..adf231ae--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-gemma4-26b.toml@@ -0,0 +1,31 @@+# GAIA eval: Gemma-4-26B-A4B-it (vLLM, 1 GPU)+[meta]+name = "gaia-gemma4-26b"+description = "GAIA on google/gemma-4-26B-A4B-it"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/gemma4-26b/gaia/"+seed = 42++[[models]]+name = "google/gemma-4-26B-A4B-it"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-gpt54.toml b/src/openjarvis/evals/configs/gaia-gpt54.tomlnew file mode 100644index 00000000..4a735d95--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-gpt54.toml@@ -0,0 +1,29 @@+# GAIA eval: gpt-5.4 (cloud)+[meta]+name = "gaia-gpt54"+description = "GAIA on gpt-5.4-2026-03-05"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 1+output_dir = "results/gaia-gpt54/"+seed = 42++[[models]]+name = "gpt-5.4-2026-03-05"+engine = "cloud"++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-nemotron-super.toml b/src/openjarvis/evals/configs/gaia-nemotron-super.tomlnew file mode 100644index 00000000..875e003c--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-nemotron-super.toml@@ -0,0 +1,30 @@+# Gaia: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8+[meta]+name = "gaia-nemotron-super"+description = "gaia on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/nemotron-super/gaia/"+seed = 42++[[models]]+name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"+engine = "vllm"+num_gpus = 2++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+max_samples = 50+agent = "monitor_operative"+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-nemotron3-super.toml b/src/openjarvis/evals/configs/gaia-nemotron3-super.tomlnew file mode 100644index 00000000..3d6d9d56--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-nemotron3-super.toml@@ -0,0 +1,31 @@+# GAIA eval: Nemotron-3-Super-120B-A12B-FP8 (SGLang on port 8001)+[meta]+name = "gaia-nemotron3-super"+description = "GAIA on Nemotron-3-Super-120B-A12B-FP8 (SGLang)"++[defaults]+temperature = 1.0+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/gaia-nemotron3-super/"+seed = 42++[[models]]+name = "nemotron"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-qwen-122b.toml b/src/openjarvis/evals/configs/gaia-qwen-122b.tomlnew file mode 100644index 00000000..94d5d4b0--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-qwen-122b.toml@@ -0,0 +1,30 @@+# Gaia: Qwen/Qwen3.5-122B-A10B-FP8+[meta]+name = "gaia-qwen-122b"+description = "gaia on Qwen/Qwen3.5-122B-A10B-FP8"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-122b/gaia/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-122B-A10B-FP8"+engine = "vllm"+num_gpus = 2++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+max_samples = 50+agent = "monitor_operative"+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-qwen-2b.toml b/src/openjarvis/evals/configs/gaia-qwen-2b.tomlnew file mode 100644index 00000000..1c68053d--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-qwen-2b.toml@@ -0,0 +1,31 @@+# GAIA eval: Qwen3.5-2B (vLLM, 1 GPU)+[meta]+name = "gaia-qwen-2b"+description = "GAIA on Qwen/Qwen3.5-2B"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-2b/gaia/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-2B"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-qwen-35b.toml b/src/openjarvis/evals/configs/gaia-qwen-35b.tomlnew file mode 100644index 00000000..9df1a672--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-qwen-35b.toml@@ -0,0 +1,30 @@+# Gaia: Qwen/Qwen3.5-35B-A3B-FP8+[meta]+name = "gaia-qwen-35b"+description = "gaia on Qwen/Qwen3.5-35B-A3B-FP8"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-35b/gaia/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-35B-A3B-FP8"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+max_samples = 50+agent = "monitor_operative"+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-qwen-9b.toml b/src/openjarvis/evals/configs/gaia-qwen-9b.tomlnew file mode 100644index 00000000..8a3a2c91--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-qwen-9b.toml@@ -0,0 +1,31 @@+# GAIA eval: Qwen3.5-9B (vLLM, 1 GPU)+[meta]+name = "gaia-qwen-9b"+description = "GAIA on Qwen/Qwen3.5-9B"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-9b/gaia/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-9B"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-qwen122b.toml b/src/openjarvis/evals/configs/gaia-qwen122b.tomlnew file mode 100644index 00000000..bb864424--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-qwen122b.toml@@ -0,0 +1,30 @@+# GAIA eval: Qwen3.5-122B-A10B-FP8 (vLLM, TP=4)+[meta]+name = "gaia-qwen122b"+description = "GAIA on Qwen/Qwen3.5-122B-A10B-FP8"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 1+output_dir = "results/gaia-qwen122b/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-122B-A10B-FP8"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-qwen35b.toml b/src/openjarvis/evals/configs/gaia-qwen35b.tomlnew file mode 100644index 00000000..be8faa4c--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-qwen35b.toml@@ -0,0 +1,30 @@+# GAIA eval: Qwen3.5-35B-A3B-FP8 (vLLM, TP=4)+[meta]+name = "gaia-qwen35b"+description = "GAIA on Qwen/Qwen3.5-35B-A3B-FP8"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 1+output_dir = "results/gaia-qwen35b/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-35B-A3B-FP8"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-rerun-claude-opus.toml b/src/openjarvis/evals/configs/gaia-rerun-claude-opus.tomlnew file mode 100644index 00000000..bde5c0dc--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-rerun-claude-opus.toml@@ -0,0 +1,29 @@+# GAIA rerun with max_turns=30: claude-opus-4-6+[meta]+name = "gaia-rerun-claude-opus"+description = "GAIA rerun with 30 max turns on claude-opus-4-6"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 1+output_dir = "results/gaia-rerun-claude-opus/"+seed = 42++[[models]]+name = "claude-opus-4-6"+engine = "cloud"++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-rerun-claude-sonnet.toml b/src/openjarvis/evals/configs/gaia-rerun-claude-sonnet.tomlnew file mode 100644index 00000000..c63575e7--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-rerun-claude-sonnet.toml@@ -0,0 +1,29 @@+# GAIA rerun with max_turns=30: claude-sonnet-4-6+[meta]+name = "gaia-rerun-claude-sonnet"+description = "GAIA rerun with 30 max turns on claude-sonnet-4-6"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 1+output_dir = "results/gaia-rerun-claude-sonnet/"+seed = 42++[[models]]+name = "claude-sonnet-4-6"+engine = "cloud"++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-rerun-glm47flash.toml b/src/openjarvis/evals/configs/gaia-rerun-glm47flash.tomlnew file mode 100644index 00000000..fe1763e0--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-rerun-glm47flash.toml@@ -0,0 +1,30 @@+# GAIA rerun with max_turns=30: zai-org/GLM-4.7-Flash+[meta]+name = "gaia-rerun-glm47flash"+description = "GAIA rerun with 30 max turns on zai-org/GLM-4.7-Flash"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 1+output_dir = "results/gaia-rerun-glm47flash/"+seed = 42++[[models]]+name = "zai-org/GLM-4.7-Flash"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-rerun-gpt5mini.toml b/src/openjarvis/evals/configs/gaia-rerun-gpt5mini.tomlnew file mode 100644index 00000000..8cdced92--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-rerun-gpt5mini.toml@@ -0,0 +1,29 @@+# GAIA rerun with max_turns=30: gpt-5-mini-2025-08-07+[meta]+name = "gaia-rerun-gpt5mini"+description = "GAIA rerun with 30 max turns on gpt-5-mini-2025-08-07"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 1+output_dir = "results/gaia-rerun-gpt5mini/"+seed = 42++[[models]]+name = "gpt-5-mini-2025-08-07"+engine = "cloud"++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-rerun-minimax-m25.toml b/src/openjarvis/evals/configs/gaia-rerun-minimax-m25.tomlnew file mode 100644index 00000000..d1a3cf78--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-rerun-minimax-m25.toml@@ -0,0 +1,30 @@+# GAIA rerun with max_turns=30: MiniMaxAI/MiniMax-M2.5+[meta]+name = "gaia-rerun-minimax-m25"+description = "GAIA rerun with 30 max turns on MiniMaxAI/MiniMax-M2.5"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 1+output_dir = "results/gaia-rerun-minimax-m25/"+seed = 42++[[models]]+name = "MiniMaxAI/MiniMax-M2.5"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-rerun-qwen122b.toml b/src/openjarvis/evals/configs/gaia-rerun-qwen122b.tomlnew file mode 100644index 00000000..286dcef5--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-rerun-qwen122b.toml@@ -0,0 +1,32 @@+# GAIA rerun with max_turns=30: Qwen/Qwen3.5-122B-A10B-FP8+[meta]+name = "gaia-rerun-qwen122b"+description = "GAIA rerun with 30 max turns on Qwen/Qwen3.5-122B-A10B-FP8"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 1+output_dir = "results/gaia-rerun-qwen122b/"+seed = 42+telemetry = true+gpu_metrics = true++[[models]]+name = "Qwen/Qwen3.5-122B-A10B-FP8"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-rerun-qwen35b.toml b/src/openjarvis/evals/configs/gaia-rerun-qwen35b.tomlnew file mode 100644index 00000000..51c51859--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-rerun-qwen35b.toml@@ -0,0 +1,30 @@+# GAIA rerun with max_turns=30: Qwen/Qwen3.5-35B-A3B-FP8+[meta]+name = "gaia-rerun-qwen35b"+description = "GAIA rerun with 30 max turns on Qwen/Qwen3.5-35B-A3B-FP8"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 1+output_dir = "results/gaia-rerun-qwen35b/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-35B-A3B-FP8"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/gaia-rerun-qwen397b.toml b/src/openjarvis/evals/configs/gaia-rerun-qwen397b.tomlnew file mode 100644index 00000000..edcbded5--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-rerun-qwen397b.toml@@ -0,0 +1,30 @@+# GAIA rerun with max_turns=30: Qwen/Qwen3.5-397B-A17B-FP8+[meta]+name = "gaia-rerun-qwen397b"+description = "GAIA rerun with 30 max turns on Qwen/Qwen3.5-397B-A17B-FP8"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 1+output_dir = "results/gaia-rerun-qwen397b/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-397B-A17B-FP8"+engine = "vllm"+num_gpus = 8++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]diff --git a/src/openjarvis/evals/configs/livecodebench-kimi-k25.toml b/src/openjarvis/evals/configs/livecodebench-kimi-k25.tomlnew file mode 100644index 00000000..655949b5--- /dev/null+++ b/src/openjarvis/evals/configs/livecodebench-kimi-k25.toml@@ -0,0 +1,39 @@+# LiveCodeBench eval: Kimi-K2.5 (Ollama, IQ4_XS GGUF)+# Competitive programming from LeetCode, AtCoder, CodeForces+# Pull model: ollama pull hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS+#+# WARNING: Kimi-K2.5 is a 1T-param MoE (32B active). ALL GGUF quants are+# sharded (smallest is 240 GB), and Ollama does not support sharded GGUFs+# yet (see github.com/ollama/ollama/issues/5245). This config is a+# placeholder -- use vLLM or wait for Ollama sharded GGUF support.+# The IQ4_XS GGUF is 547 GB and would require 7x H100 80GB GPUs.++[meta]+name = "livecodebench-kimi-k25"+description = "LiveCodeBench on Kimi-K2.5 IQ4_XS via Ollama"++[defaults]+temperature = 0.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/kimi-k25-ollama/livecodebench/"+seed = 42++[[models]]+name = "hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS"+engine = "ollama"+param_count_b = 1000.0+active_params_b = 32.0+num_gpus = 7++[[benchmarks]]+name = "livecodebench"+backend = "jarvis-direct"+max_samples = 20diff --git a/src/openjarvis/evals/configs/livecodebench-minimax-m25.toml b/src/openjarvis/evals/configs/livecodebench-minimax-m25.tomlnew file mode 100644index 00000000..73cf2930--- /dev/null+++ b/src/openjarvis/evals/configs/livecodebench-minimax-m25.toml@@ -0,0 +1,36 @@+# LiveCodeBench eval: MiniMax-M2.5 (Ollama, UD-TQ1_0 GGUF ~55 GB, GPUs 2-3)+# Competitive programming from LeetCode, AtCoder, CodeForces+# Pull model: ollama pull hf.co/unsloth/MiniMax-M2.5-GGUF:latest+# NOTE: Q4_K_M and higher quants are sharded GGUFs which Ollama does not+# support yet (see github.com/ollama/ollama/issues/5245). The "latest" tag+# pulls the largest non-sharded quant: UD-TQ1_0 (1-bit, ~55 GB).++[meta]+name = "livecodebench-minimax-m25"+description = "LiveCodeBench on MiniMax-M2.5 UD-TQ1_0 via Ollama"++[defaults]+temperature = 0.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/minimax-m25-ollama/livecodebench/"+seed = 42++[[models]]+name = "hf.co/unsloth/MiniMax-M2.5-GGUF:latest"+engine = "ollama"+param_count_b = 229.0+active_params_b = 45.6+num_gpus = 2++[[benchmarks]]+name = "livecodebench"+backend = "jarvis-direct"+max_samples = 20diff --git a/src/openjarvis/evals/configs/livecodebench-nemotron-super.toml b/src/openjarvis/evals/configs/livecodebench-nemotron-super.tomlnew file mode 100644index 00000000..30b8697b--- /dev/null+++ b/src/openjarvis/evals/configs/livecodebench-nemotron-super.toml@@ -0,0 +1,28 @@+# Livecodebench: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8+[meta]+name = "livecodebench-nemotron-super"+description = "livecodebench on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"++[defaults]+temperature = 0.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/nemotron-super/livecodebench/"+seed = 42++[[models]]+name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"+engine = "vllm"+num_gpus = 2++[[benchmarks]]+name = "livecodebench"+backend = "jarvis-direct"+max_samples = 20diff --git a/src/openjarvis/evals/configs/livecodebench-qwen-122b.toml b/src/openjarvis/evals/configs/livecodebench-qwen-122b.tomlnew file mode 100644index 00000000..3ac0c749--- /dev/null+++ b/src/openjarvis/evals/configs/livecodebench-qwen-122b.toml@@ -0,0 +1,28 @@+# Livecodebench: Qwen/Qwen3.5-122B-A10B-FP8+[meta]+name = "livecodebench-qwen-122b"+description = "livecodebench on Qwen/Qwen3.5-122B-A10B-FP8"++[defaults]+temperature = 0.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-122b/livecodebench/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-122B-A10B-FP8"+engine = "vllm"+num_gpus = 2++[[benchmarks]]+name = "livecodebench"+backend = "jarvis-direct"+max_samples = 20diff --git a/src/openjarvis/evals/configs/livecodebench-qwen-35b.toml b/src/openjarvis/evals/configs/livecodebench-qwen-35b.tomlnew file mode 100644index 00000000..281a5e84--- /dev/null+++ b/src/openjarvis/evals/configs/livecodebench-qwen-35b.toml@@ -0,0 +1,28 @@+# Livecodebench: Qwen/Qwen3.5-35B-A3B-FP8+[meta]+name = "livecodebench-qwen-35b"+description = "livecodebench on Qwen/Qwen3.5-35B-A3B-FP8"++[defaults]+temperature = 0.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-35b/livecodebench/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-35B-A3B-FP8"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "livecodebench"+backend = "jarvis-direct"+max_samples = 20diff --git a/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml b/src/openjarvis/evals/configs/liveresearch-gemma4-26b.tomlnew file mode 100644index 00000000..a398c7d0--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml@@ -0,0 +1,31 @@+# LiveResearchBench: Gemma-4-26B-A4B-it (vLLM, 1 GPU)+[meta]+name = "liveresearch-gemma4-26b"+description = "LiveResearchBench on google/gemma-4-26B-A4B-it"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/gemma4-26b/liveresearch/"+seed = 42++[[models]]+name = "google/gemma-4-26B-A4B-it"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-nemotron-super.toml b/src/openjarvis/evals/configs/liveresearch-nemotron-super.tomlnew file mode 100644index 00000000..f14956fa--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-nemotron-super.toml@@ -0,0 +1,30 @@+# Liveresearch: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8+[meta]+name = "liveresearch-nemotron-super"+description = "liveresearch on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/nemotron-super/liveresearch/"+seed = 42++[[models]]+name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"+engine = "vllm"+num_gpus = 2++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+max_samples = 50+agent = "monitor_operative"+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-122b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-122b.tomlnew file mode 100644index 00000000..0e316fa2--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-qwen-122b.toml@@ -0,0 +1,30 @@+# Liveresearch: Qwen/Qwen3.5-122B-A10B-FP8+[meta]+name = "liveresearch-qwen-122b"+description = "liveresearch on Qwen/Qwen3.5-122B-A10B-FP8"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-122b/liveresearch/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-122B-A10B-FP8"+engine = "vllm"+num_gpus = 2++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+max_samples = 50+agent = "monitor_operative"+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b.tomlnew file mode 100644index 00000000..d0beff38--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml@@ -0,0 +1,31 @@+# LiveResearchBench: Qwen3.5-2B (vLLM, 1 GPU)+[meta]+name = "liveresearch-qwen-2b"+description = "LiveResearchBench on Qwen/Qwen3.5-2B"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-2b/liveresearch/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-2B"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-35b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-35b.tomlnew file mode 100644index 00000000..483229ee--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-qwen-35b.toml@@ -0,0 +1,30 @@+# Liveresearch: Qwen/Qwen3.5-35B-A3B-FP8+[meta]+name = "liveresearch-qwen-35b"+description = "liveresearch on Qwen/Qwen3.5-35B-A3B-FP8"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-35b/liveresearch/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-35B-A3B-FP8"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+max_samples = 50+agent = "monitor_operative"+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b.tomlnew file mode 100644index 00000000..002e02ae--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml@@ -0,0 +1,31 @@+# LiveResearchBench: Qwen3.5-9B (vLLM, 1 GPU)+[meta]+name = "liveresearch-qwen-9b"+description = "LiveResearchBench on Qwen/Qwen3.5-9B"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-9b/liveresearch/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-9B"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/pinchbench-kimi-k25.toml b/src/openjarvis/evals/configs/pinchbench-kimi-k25.tomlnew file mode 100644index 00000000..e5ef8667--- /dev/null+++ b/src/openjarvis/evals/configs/pinchbench-kimi-k25.toml@@ -0,0 +1,44 @@+# PinchBench eval: Kimi-K2.5 (Ollama, IQ4_XS GGUF)+# Agent: native_openhands -- all PinchBench-required tools enabled+# Pull model: ollama pull hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS+#+# WARNING: Kimi-K2.5 is a 1T-param MoE (32B active). ALL GGUF quants are+# sharded (smallest is 240 GB), and Ollama does not support sharded GGUFs+# yet (see github.com/ollama/ollama/issues/5245). This config is a+# placeholder -- use vLLM or wait for Ollama sharded GGUF support.+# The IQ4_XS GGUF is 547 GB and would require 7x H100 80GB GPUs.++[meta]+name = "pinchbench-kimi-k25"+description = "PinchBench on Kimi-K2.5 IQ4_XS via Ollama"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "claude-opus-4-5"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/kimi-k25-ollama/pinchbench/"+seed = 42++[[models]]+name = "hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS"+engine = "ollama"+param_count_b = 1000.0+active_params_b = 32.0+num_gpus = 7++[[benchmarks]]+name = "pinchbench"+backend = "jarvis-agent"+agent = "native_openhands"+tools = [+ "think", "file_read", "file_write", "web_search", "shell_exec",+ "code_interpreter", "browser_navigate", "image_generate",+ "calculator", "http_request", "pdf_extract",+]diff --git a/src/openjarvis/evals/configs/pinchbench-minimax-m25.toml b/src/openjarvis/evals/configs/pinchbench-minimax-m25.tomlnew file mode 100644index 00000000..e9662b3a--- /dev/null+++ b/src/openjarvis/evals/configs/pinchbench-minimax-m25.toml@@ -0,0 +1,41 @@+# PinchBench eval: MiniMax-M2.5 (Ollama, UD-TQ1_0 GGUF ~55 GB, GPUs 2-3)+# Agent: native_openhands -- all PinchBench-required tools enabled+# Pull model: ollama pull hf.co/unsloth/MiniMax-M2.5-GGUF:latest+# NOTE: Q4_K_M and higher quants are sharded GGUFs which Ollama does not+# support yet (see github.com/ollama/ollama/issues/5245). The "latest" tag+# pulls the largest non-sharded quant: UD-TQ1_0 (1-bit, ~55 GB).++[meta]+name = "pinchbench-minimax-m25"+description = "PinchBench on MiniMax-M2.5 UD-TQ1_0 via Ollama"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "claude-opus-4-5"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/minimax-m25-ollama/pinchbench/"+seed = 42++[[models]]+name = "hf.co/unsloth/MiniMax-M2.5-GGUF:latest"+engine = "ollama"+param_count_b = 229.0+active_params_b = 45.6+num_gpus = 2++[[benchmarks]]+name = "pinchbench"+backend = "jarvis-agent"+agent = "native_openhands"+tools = [+ "think", "file_read", "file_write", "web_search", "shell_exec",+ "code_interpreter", "browser_navigate", "image_generate",+ "calculator", "http_request", "pdf_extract",+]diff --git a/src/openjarvis/evals/configs/pinchbench-nemotron-super.toml b/src/openjarvis/evals/configs/pinchbench-nemotron-super.tomlnew file mode 100644index 00000000..c66090ea--- /dev/null+++ b/src/openjarvis/evals/configs/pinchbench-nemotron-super.toml@@ -0,0 +1,34 @@+# Pinchbench: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8+[meta]+name = "pinchbench-nemotron-super"+description = "pinchbench on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/nemotron-super/pinchbench/"+seed = 42++[[models]]+name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"+engine = "vllm"+num_gpus = 2++[[benchmarks]]+name = "pinchbench"+backend = "jarvis-agent"+max_samples = 23+agent = "native_openhands"+tools = [+ "think", "file_read", "file_write", "web_search", "shell_exec",+ "code_interpreter", "browser_navigate", "image_generate",+ "calculator", "http_request", "pdf_extract",+]diff --git a/src/openjarvis/evals/configs/pinchbench-qwen-122b.toml b/src/openjarvis/evals/configs/pinchbench-qwen-122b.tomlnew file mode 100644index 00000000..5a91cc93--- /dev/null+++ b/src/openjarvis/evals/configs/pinchbench-qwen-122b.toml@@ -0,0 +1,34 @@+# Pinchbench: Qwen/Qwen3.5-122B-A10B-FP8+[meta]+name = "pinchbench-qwen-122b"+description = "pinchbench on Qwen/Qwen3.5-122B-A10B-FP8"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-122b/pinchbench/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-122B-A10B-FP8"+engine = "vllm"+num_gpus = 2++[[benchmarks]]+name = "pinchbench"+backend = "jarvis-agent"+max_samples = 23+agent = "native_openhands"+tools = [+ "think", "file_read", "file_write", "web_search", "shell_exec",+ "code_interpreter", "browser_navigate", "image_generate",+ "calculator", "http_request", "pdf_extract",+]diff --git a/src/openjarvis/evals/configs/pinchbench-qwen-35b.toml b/src/openjarvis/evals/configs/pinchbench-qwen-35b.tomlnew file mode 100644index 00000000..1f2188ee--- /dev/null+++ b/src/openjarvis/evals/configs/pinchbench-qwen-35b.toml@@ -0,0 +1,34 @@+# Pinchbench: Qwen/Qwen3.5-35B-A3B-FP8+[meta]+name = "pinchbench-qwen-35b"+description = "pinchbench on Qwen/Qwen3.5-35B-A3B-FP8"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-35b/pinchbench/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-35B-A3B-FP8"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "pinchbench"+backend = "jarvis-agent"+max_samples = 23+agent = "native_openhands"+tools = [+ "think", "file_read", "file_write", "web_search", "shell_exec",+ "code_interpreter", "browser_navigate", "image_generate",+ "calculator", "http_request", "pdf_extract",+]diff --git a/src/openjarvis/evals/configs/taubench-kimi-k25.toml b/src/openjarvis/evals/configs/taubench-kimi-k25.tomlnew file mode 100644index 00000000..639f3f05--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-kimi-k25.toml@@ -0,0 +1,40 @@+# TauBench V2 eval: Kimi-K2.5 (Ollama, IQ4_XS GGUF)+# Multi-turn customer service benchmark -- airline + retail splits+# Pull model: ollama pull hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS+#+# WARNING: Kimi-K2.5 is a 1T-param MoE (32B active). ALL GGUF quants are+# sharded (smallest is 240 GB), and Ollama does not support sharded GGUFs+# yet (see github.com/ollama/ollama/issues/5245). This config is a+# placeholder -- use vLLM or wait for Ollama sharded GGUF support.+# The IQ4_XS GGUF is 547 GB and would require 7x H100 80GB GPUs.++[meta]+name = "taubench-kimi-k25"+description = "TauBench V2 on Kimi-K2.5 IQ4_XS via Ollama"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/kimi-k25-ollama/taubench/"+seed = 42++[[models]]+name = "hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS"+engine = "ollama"+param_count_b = 1000.0+active_params_b = 32.0+num_gpus = 7++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+max_samples = 20+split = "airline,retail"diff --git a/src/openjarvis/evals/configs/taubench-minimax-m25.toml b/src/openjarvis/evals/configs/taubench-minimax-m25.tomlnew file mode 100644index 00000000..c962efd8--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-minimax-m25.toml@@ -0,0 +1,37 @@+# TauBench V2 eval: MiniMax-M2.5 (Ollama, UD-TQ1_0 GGUF ~55 GB, GPUs 2-3)+# Multi-turn customer service benchmark -- airline + retail splits+# Pull model: ollama pull hf.co/unsloth/MiniMax-M2.5-GGUF:latest+# NOTE: Q4_K_M and higher quants are sharded GGUFs which Ollama does not+# support yet (see github.com/ollama/ollama/issues/5245). The "latest" tag+# pulls the largest non-sharded quant: UD-TQ1_0 (1-bit, ~55 GB).++[meta]+name = "taubench-minimax-m25"+description = "TauBench V2 on MiniMax-M2.5 UD-TQ1_0 via Ollama"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/minimax-m25-ollama/taubench/"+seed = 42++[[models]]+name = "hf.co/unsloth/MiniMax-M2.5-GGUF:latest"+engine = "ollama"+param_count_b = 229.0+active_params_b = 45.6+num_gpus = 2++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+max_samples = 20+split = "airline,retail"diff --git a/src/openjarvis/evals/configs/taubench-nemotron-super.toml b/src/openjarvis/evals/configs/taubench-nemotron-super.tomlnew file mode 100644index 00000000..f7f5304b--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-nemotron-super.toml@@ -0,0 +1,29 @@+# Taubench: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8+[meta]+name = "taubench-nemotron-super"+description = "taubench on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/nemotron-super/taubench/"+seed = 42++[[models]]+name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"+engine = "vllm"+num_gpus = 2++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+max_samples = 20+split = "airline,retail"diff --git a/src/openjarvis/evals/configs/taubench-qwen-122b.toml b/src/openjarvis/evals/configs/taubench-qwen-122b.tomlnew file mode 100644index 00000000..a0f4329a--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-qwen-122b.toml@@ -0,0 +1,29 @@+# Taubench: Qwen/Qwen3.5-122B-A10B-FP8+[meta]+name = "taubench-qwen-122b"+description = "taubench on Qwen/Qwen3.5-122B-A10B-FP8"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-122b/taubench/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-122B-A10B-FP8"+engine = "vllm"+num_gpus = 2++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+max_samples = 20+split = "airline,retail"diff --git a/src/openjarvis/evals/configs/taubench-qwen-35b.toml b/src/openjarvis/evals/configs/taubench-qwen-35b.tomlnew file mode 100644index 00000000..d902364b--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-qwen-35b.toml@@ -0,0 +1,29 @@+# Taubench: Qwen/Qwen3.5-35B-A3B-FP8+[meta]+name = "taubench-qwen-35b"+description = "taubench on Qwen/Qwen3.5-35B-A3B-FP8"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-35b/taubench/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-35B-A3B-FP8"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+max_samples = 20+split = "airline,retail"diff --git a/src/openjarvis/evals/configs/taubench-telecom-claude.toml b/src/openjarvis/evals/configs/taubench-telecom-claude.tomlnew file mode 100644index 00000000..2710b225--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-telecom-claude.toml@@ -0,0 +1,27 @@+# TauBench V2 Telecom: Claude Opus 4.6+[meta]+name = "taubench-telecom-claude"+description = "TauBench V2 telecom on claude-opus-4-6 (test split, pass^3)"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/taubench-telecom-claude/"+seed = 42++[[models]]+name = "claude-opus-4-6"+engine = "cloud"++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+split = "telecom"diff --git a/src/openjarvis/evals/configs/taubench-telecom-gemma4-26b.toml b/src/openjarvis/evals/configs/taubench-telecom-gemma4-26b.tomlnew file mode 100644index 00000000..380ab271--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-telecom-gemma4-26b.toml@@ -0,0 +1,29 @@+# TauBench V2 Telecom: Gemma-4-26B-A4B-it (vLLM, 1 GPU)+[meta]+name = "taubench-telecom-gemma4-26b"+description = "TauBench V2 Telecom domain on google/gemma-4-26B-A4B-it"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/gemma4-26b/taubench-telecom/"+seed = 42++[[models]]+name = "google/gemma-4-26B-A4B-it"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+max_samples = 20+split = "telecom"diff --git a/src/openjarvis/evals/configs/taubench-telecom-gpt54.toml b/src/openjarvis/evals/configs/taubench-telecom-gpt54.tomlnew file mode 100644index 00000000..1dbb3f3a--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-telecom-gpt54.toml@@ -0,0 +1,27 @@+# TauBench V2 Telecom: GPT-5.4+[meta]+name = "taubench-telecom-gpt54"+description = "TauBench V2 telecom on gpt-5.4 (test split, pass^3)"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/taubench-telecom-gpt54/"+seed = 42++[[models]]+name = "gpt-5.4-2026-03-05"+engine = "cloud"++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+split = "telecom"diff --git a/src/openjarvis/evals/configs/taubench-telecom-nemotron-super.toml b/src/openjarvis/evals/configs/taubench-telecom-nemotron-super.tomlnew file mode 100644index 00000000..26cffbd2--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-telecom-nemotron-super.toml@@ -0,0 +1,29 @@+# Taubench: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8+[meta]+name = "taubench-telecom-nemotron-super"+description = "taubench on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/nemotron-super/taubench-telecom/"+seed = 42++[[models]]+name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"+engine = "vllm"+num_gpus = 2++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+max_samples = 20+split = "telecom"diff --git a/src/openjarvis/evals/configs/taubench-telecom-nemotron.toml b/src/openjarvis/evals/configs/taubench-telecom-nemotron.tomlnew file mode 100644index 00000000..d71b37a2--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-telecom-nemotron.toml@@ -0,0 +1,28 @@+# TauBench V2 Telecom: Nemotron-3-Super (SGLang port 8001)+[meta]+name = "taubench-telecom-nemotron"+description = "TauBench V2 telecom on Nemotron-3-Super (test split, pass^3)"++[defaults]+temperature = 1.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/taubench-telecom-nemotron/"+seed = 42++[[models]]+name = "nemotron"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+split = "telecom"diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen-122b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen-122b.tomlnew file mode 100644index 00000000..849fb30e--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-telecom-qwen-122b.toml@@ -0,0 +1,29 @@+# Taubench: Qwen/Qwen3.5-122B-A10B-FP8+[meta]+name = "taubench-telecom-qwen-122b"+description = "taubench on Qwen/Qwen3.5-122B-A10B-FP8"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-122b/taubench-telecom/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-122B-A10B-FP8"+engine = "vllm"+num_gpus = 2++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+max_samples = 20+split = "telecom"diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen-35b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen-35b.tomlnew file mode 100644index 00000000..ccf5c8f6--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-telecom-qwen-35b.toml@@ -0,0 +1,29 @@+# Taubench: Qwen/Qwen3.5-35B-A3B-FP8+[meta]+name = "taubench-telecom-qwen-35b"+description = "taubench on Qwen/Qwen3.5-35B-A3B-FP8"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-35b/taubench-telecom/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-35B-A3B-FP8"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+max_samples = 20+split = "telecom"diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen27b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen27b.tomlnew file mode 100644index 00000000..c4a2de6f--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-telecom-qwen27b.toml@@ -0,0 +1,29 @@+# TauBench V2 Telecom: Qwen3.5-27B-FP8 (vLLM, 1 GPU)+[meta]+name = "taubench-telecom-qwen27b"+description = "TauBench V2 Telecom domain on Qwen/Qwen3.5-27B-FP8"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-27b/taubench-telecom/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-27B-FP8"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+max_samples = 20+split = "telecom"diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen2b.tomlnew file mode 100644index 00000000..d1fd6023--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml@@ -0,0 +1,29 @@+# TauBench V2 Telecom: Qwen3.5-2B (vLLM, 1 GPU)+[meta]+name = "taubench-telecom-qwen2b"+description = "TauBench V2 Telecom domain on Qwen/Qwen3.5-2B"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-2b/taubench-telecom/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-2B"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+max_samples = 20+split = "telecom"diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen9b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen9b.tomlnew file mode 100644index 00000000..85d8691e--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-telecom-qwen9b.toml@@ -0,0 +1,29 @@+# TauBench V2 Telecom: Qwen3.5-9B (vLLM, 1 GPU)+[meta]+name = "taubench-telecom-qwen9b"+description = "TauBench V2 Telecom domain on Qwen/Qwen3.5-9B"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-9b/taubench-telecom/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-9B"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+max_samples = 20+split = "telecom"diff --git a/src/openjarvis/evals/configs/toolcall15-kimi-k25.toml b/src/openjarvis/evals/configs/toolcall15-kimi-k25.tomlnew file mode 100644index 00000000..2f0596bc--- /dev/null+++ b/src/openjarvis/evals/configs/toolcall15-kimi-k25.toml@@ -0,0 +1,38 @@+# ToolCall-15 eval: Kimi-K2.5 (Ollama, IQ4_XS GGUF)+# Lightweight tool calling benchmark -- 15 scenarios, 5 categories+# Pull model: ollama pull hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS+#+# WARNING: Kimi-K2.5 is a 1T-param MoE (32B active). ALL GGUF quants are+# sharded (smallest is 240 GB), and Ollama does not support sharded GGUFs+# yet (see github.com/ollama/ollama/issues/5245). This config is a+# placeholder -- use vLLM or wait for Ollama sharded GGUF support.+# The IQ4_XS GGUF is 547 GB and would require 7x H100 80GB GPUs.++[meta]+name = "toolcall15-kimi-k25"+description = "ToolCall-15 on Kimi-K2.5 IQ4_XS via Ollama"++[defaults]+temperature = 0.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/kimi-k25-ollama/toolcall15/"+seed = 42++[[models]]+name = "hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS"+engine = "ollama"+param_count_b = 1000.0+active_params_b = 32.0+num_gpus = 7++[[benchmarks]]+name = "toolcall15"+backend = "jarvis-direct"diff --git a/src/openjarvis/evals/configs/toolcall15-minimax-m25.toml b/src/openjarvis/evals/configs/toolcall15-minimax-m25.tomlnew file mode 100644index 00000000..6d087419--- /dev/null+++ b/src/openjarvis/evals/configs/toolcall15-minimax-m25.toml@@ -0,0 +1,35 @@+# ToolCall-15 eval: MiniMax-M2.5 (Ollama, UD-TQ1_0 GGUF ~55 GB, GPUs 2-3)+# Lightweight tool calling benchmark -- 15 scenarios, 5 categories+# Pull model: ollama pull hf.co/unsloth/MiniMax-M2.5-GGUF:latest+# NOTE: Q4_K_M and higher quants are sharded GGUFs which Ollama does not+# support yet (see github.com/ollama/ollama/issues/5245). The "latest" tag+# pulls the largest non-sharded quant: UD-TQ1_0 (1-bit, ~55 GB).++[meta]+name = "toolcall15-minimax-m25"+description = "ToolCall-15 on MiniMax-M2.5 UD-TQ1_0 via Ollama"++[defaults]+temperature = 0.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/minimax-m25-ollama/toolcall15/"+seed = 42++[[models]]+name = "hf.co/unsloth/MiniMax-M2.5-GGUF:latest"+engine = "ollama"+param_count_b = 229.0+active_params_b = 45.6+num_gpus = 2++[[benchmarks]]+name = "toolcall15"+backend = "jarvis-direct"diff --git a/src/openjarvis/evals/configs/toolcall15-nemotron-super.toml b/src/openjarvis/evals/configs/toolcall15-nemotron-super.tomlnew file mode 100644index 00000000..9b3fba0f--- /dev/null+++ b/src/openjarvis/evals/configs/toolcall15-nemotron-super.toml@@ -0,0 +1,28 @@+# Toolcall15: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8+[meta]+name = "toolcall15-nemotron-super"+description = "toolcall15 on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"++[defaults]+temperature = 0.6+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/nemotron-super/toolcall15/"+seed = 42++[[models]]+name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"+engine = "vllm"+num_gpus = 2++[[benchmarks]]+name = "toolcall15"+backend = "jarvis-direct"+max_samples = 15diff --git a/src/openjarvis/evals/configs/toolcall15-qwen-122b.toml b/src/openjarvis/evals/configs/toolcall15-qwen-122b.tomlnew file mode 100644index 00000000..b88527e4--- /dev/null+++ b/src/openjarvis/evals/configs/toolcall15-qwen-122b.toml@@ -0,0 +1,28 @@+# Toolcall15: Qwen/Qwen3.5-122B-A10B-FP8+[meta]+name = "toolcall15-qwen-122b"+description = "toolcall15 on Qwen/Qwen3.5-122B-A10B-FP8"++[defaults]+temperature = 0.6+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-122b/toolcall15/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-122B-A10B-FP8"+engine = "vllm"+num_gpus = 2++[[benchmarks]]+name = "toolcall15"+backend = "jarvis-direct"+max_samples = 15diff --git a/src/openjarvis/evals/configs/toolcall15-qwen-35b.toml b/src/openjarvis/evals/configs/toolcall15-qwen-35b.tomlnew file mode 100644index 00000000..7dd7ccee--- /dev/null+++ b/src/openjarvis/evals/configs/toolcall15-qwen-35b.toml@@ -0,0 +1,28 @@+# Toolcall15: Qwen/Qwen3.5-35B-A3B-FP8+[meta]+name = "toolcall15-qwen-35b"+description = "toolcall15 on Qwen/Qwen3.5-35B-A3B-FP8"++[defaults]+temperature = 0.6+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-35b/toolcall15/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-35B-A3B-FP8"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "toolcall15"+backend = "jarvis-direct"+max_samples = 15