Skip to content

Personal AI, On Personal Devices

Python

add eval configs (#208)

Avanika Narayan41ddab9099ec
diff --git a/.gitignore b/.gitignore
index a86fd343..a993fcf3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -90,7 +90,4 @@ CLAUDE.md
research_mining_*
.python-version
src/openjarvis/channels/whatsapp_baileys_bridge/node_modules/
-
-# SQLite in-memory artifacts
-:memory:
-MagicMock/
+scratch/
diff --git a/src/openjarvis/evals/configs/gaia-gemini-31-pro.toml b/src/openjarvis/evals/configs/gaia-gemini-31-pro.toml
new file mode 100644
index 00000000..1410012e
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-gemini-31-pro.toml
@@ -0,0 +1,29 @@
+# GAIA eval: gemini-3.1-pro-preview (cloud)
+[meta]
+name = "gaia-gemini-31-pro"
+description = "GAIA on gemini-3.1-pro-preview"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/gaia-gemini-31-pro/"
+seed = 42
+
+[[models]]
+name = "gemini-3.1-pro-preview"
+engine = "cloud"
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-gemma4-26b.toml b/src/openjarvis/evals/configs/gaia-gemma4-26b.toml
new file mode 100644
index 00000000..adf231ae
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-gemma4-26b.toml
@@ -0,0 +1,31 @@
+# GAIA eval: Gemma-4-26B-A4B-it (vLLM, 1 GPU)
+[meta]
+name = "gaia-gemma4-26b"
+description = "GAIA on google/gemma-4-26B-A4B-it"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/gemma4-26b/gaia/"
+seed = 42
+
+[[models]]
+name = "google/gemma-4-26B-A4B-it"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-gpt54.toml b/src/openjarvis/evals/configs/gaia-gpt54.toml
new file mode 100644
index 00000000..4a735d95
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-gpt54.toml
@@ -0,0 +1,29 @@
+# GAIA eval: gpt-5.4 (cloud)
+[meta]
+name = "gaia-gpt54"
+description = "GAIA on gpt-5.4-2026-03-05"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/gaia-gpt54/"
+seed = 42
+
+[[models]]
+name = "gpt-5.4-2026-03-05"
+engine = "cloud"
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-nemotron-super.toml b/src/openjarvis/evals/configs/gaia-nemotron-super.toml
new file mode 100644
index 00000000..875e003c
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-nemotron-super.toml
@@ -0,0 +1,30 @@
+# Gaia: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8
+[meta]
+name = "gaia-nemotron-super"
+description = "gaia on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/nemotron-super/gaia/"
+seed = 42
+
+[[models]]
+name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
+engine = "vllm"
+num_gpus = 2
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+max_samples = 50
+agent = "monitor_operative"
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-nemotron3-super.toml b/src/openjarvis/evals/configs/gaia-nemotron3-super.toml
new file mode 100644
index 00000000..3d6d9d56
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-nemotron3-super.toml
@@ -0,0 +1,31 @@
+# GAIA eval: Nemotron-3-Super-120B-A12B-FP8 (SGLang on port 8001)
+[meta]
+name = "gaia-nemotron3-super"
+description = "GAIA on Nemotron-3-Super-120B-A12B-FP8 (SGLang)"
+
+[defaults]
+temperature = 1.0
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/gaia-nemotron3-super/"
+seed = 42
+
+[[models]]
+name = "nemotron"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-qwen-122b.toml b/src/openjarvis/evals/configs/gaia-qwen-122b.toml
new file mode 100644
index 00000000..94d5d4b0
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-qwen-122b.toml
@@ -0,0 +1,30 @@
+# Gaia: Qwen/Qwen3.5-122B-A10B-FP8
+[meta]
+name = "gaia-qwen-122b"
+description = "gaia on Qwen/Qwen3.5-122B-A10B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-122b/gaia/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-122B-A10B-FP8"
+engine = "vllm"
+num_gpus = 2
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+max_samples = 50
+agent = "monitor_operative"
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-qwen-2b.toml b/src/openjarvis/evals/configs/gaia-qwen-2b.toml
new file mode 100644
index 00000000..1c68053d
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-qwen-2b.toml
@@ -0,0 +1,31 @@
+# GAIA eval: Qwen3.5-2B (vLLM, 1 GPU)
+[meta]
+name = "gaia-qwen-2b"
+description = "GAIA on Qwen/Qwen3.5-2B"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-2b/gaia/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-2B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-qwen-35b.toml b/src/openjarvis/evals/configs/gaia-qwen-35b.toml
new file mode 100644
index 00000000..9df1a672
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-qwen-35b.toml
@@ -0,0 +1,30 @@
+# Gaia: Qwen/Qwen3.5-35B-A3B-FP8
+[meta]
+name = "gaia-qwen-35b"
+description = "gaia on Qwen/Qwen3.5-35B-A3B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-35b/gaia/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-35B-A3B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+max_samples = 50
+agent = "monitor_operative"
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-qwen-9b.toml b/src/openjarvis/evals/configs/gaia-qwen-9b.toml
new file mode 100644
index 00000000..8a3a2c91
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-qwen-9b.toml
@@ -0,0 +1,31 @@
+# GAIA eval: Qwen3.5-9B (vLLM, 1 GPU)
+[meta]
+name = "gaia-qwen-9b"
+description = "GAIA on Qwen/Qwen3.5-9B"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-9b/gaia/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-9B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-qwen122b.toml b/src/openjarvis/evals/configs/gaia-qwen122b.toml
new file mode 100644
index 00000000..bb864424
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-qwen122b.toml
@@ -0,0 +1,30 @@
+# GAIA eval: Qwen3.5-122B-A10B-FP8 (vLLM, TP=4)
+[meta]
+name = "gaia-qwen122b"
+description = "GAIA on Qwen/Qwen3.5-122B-A10B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/gaia-qwen122b/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-122B-A10B-FP8"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-qwen35b.toml b/src/openjarvis/evals/configs/gaia-qwen35b.toml
new file mode 100644
index 00000000..be8faa4c
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-qwen35b.toml
@@ -0,0 +1,30 @@
+# GAIA eval: Qwen3.5-35B-A3B-FP8 (vLLM, TP=4)
+[meta]
+name = "gaia-qwen35b"
+description = "GAIA on Qwen/Qwen3.5-35B-A3B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/gaia-qwen35b/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-35B-A3B-FP8"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-rerun-claude-opus.toml b/src/openjarvis/evals/configs/gaia-rerun-claude-opus.toml
new file mode 100644
index 00000000..bde5c0dc
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-rerun-claude-opus.toml
@@ -0,0 +1,29 @@
+# GAIA rerun with max_turns=30: claude-opus-4-6
+[meta]
+name = "gaia-rerun-claude-opus"
+description = "GAIA rerun with 30 max turns on claude-opus-4-6"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/gaia-rerun-claude-opus/"
+seed = 42
+
+[[models]]
+name = "claude-opus-4-6"
+engine = "cloud"
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-rerun-claude-sonnet.toml b/src/openjarvis/evals/configs/gaia-rerun-claude-sonnet.toml
new file mode 100644
index 00000000..c63575e7
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-rerun-claude-sonnet.toml
@@ -0,0 +1,29 @@
+# GAIA rerun with max_turns=30: claude-sonnet-4-6
+[meta]
+name = "gaia-rerun-claude-sonnet"
+description = "GAIA rerun with 30 max turns on claude-sonnet-4-6"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/gaia-rerun-claude-sonnet/"
+seed = 42
+
+[[models]]
+name = "claude-sonnet-4-6"
+engine = "cloud"
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-rerun-glm47flash.toml b/src/openjarvis/evals/configs/gaia-rerun-glm47flash.toml
new file mode 100644
index 00000000..fe1763e0
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-rerun-glm47flash.toml
@@ -0,0 +1,30 @@
+# GAIA rerun with max_turns=30: zai-org/GLM-4.7-Flash
+[meta]
+name = "gaia-rerun-glm47flash"
+description = "GAIA rerun with 30 max turns on zai-org/GLM-4.7-Flash"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/gaia-rerun-glm47flash/"
+seed = 42
+
+[[models]]
+name = "zai-org/GLM-4.7-Flash"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-rerun-gpt5mini.toml b/src/openjarvis/evals/configs/gaia-rerun-gpt5mini.toml
new file mode 100644
index 00000000..8cdced92
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-rerun-gpt5mini.toml
@@ -0,0 +1,29 @@
+# GAIA rerun with max_turns=30: gpt-5-mini-2025-08-07
+[meta]
+name = "gaia-rerun-gpt5mini"
+description = "GAIA rerun with 30 max turns on gpt-5-mini-2025-08-07"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/gaia-rerun-gpt5mini/"
+seed = 42
+
+[[models]]
+name = "gpt-5-mini-2025-08-07"
+engine = "cloud"
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-rerun-minimax-m25.toml b/src/openjarvis/evals/configs/gaia-rerun-minimax-m25.toml
new file mode 100644
index 00000000..d1a3cf78
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-rerun-minimax-m25.toml
@@ -0,0 +1,30 @@
+# GAIA rerun with max_turns=30: MiniMaxAI/MiniMax-M2.5
+[meta]
+name = "gaia-rerun-minimax-m25"
+description = "GAIA rerun with 30 max turns on MiniMaxAI/MiniMax-M2.5"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/gaia-rerun-minimax-m25/"
+seed = 42
+
+[[models]]
+name = "MiniMaxAI/MiniMax-M2.5"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-rerun-qwen122b.toml b/src/openjarvis/evals/configs/gaia-rerun-qwen122b.toml
new file mode 100644
index 00000000..286dcef5
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-rerun-qwen122b.toml
@@ -0,0 +1,32 @@
+# GAIA rerun with max_turns=30: Qwen/Qwen3.5-122B-A10B-FP8
+[meta]
+name = "gaia-rerun-qwen122b"
+description = "GAIA rerun with 30 max turns on Qwen/Qwen3.5-122B-A10B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/gaia-rerun-qwen122b/"
+seed = 42
+telemetry = true
+gpu_metrics = true
+
+[[models]]
+name = "Qwen/Qwen3.5-122B-A10B-FP8"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-rerun-qwen35b.toml b/src/openjarvis/evals/configs/gaia-rerun-qwen35b.toml
new file mode 100644
index 00000000..51c51859
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-rerun-qwen35b.toml
@@ -0,0 +1,30 @@
+# GAIA rerun with max_turns=30: Qwen/Qwen3.5-35B-A3B-FP8
+[meta]
+name = "gaia-rerun-qwen35b"
+description = "GAIA rerun with 30 max turns on Qwen/Qwen3.5-35B-A3B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/gaia-rerun-qwen35b/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-35B-A3B-FP8"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/gaia-rerun-qwen397b.toml b/src/openjarvis/evals/configs/gaia-rerun-qwen397b.toml
new file mode 100644
index 00000000..edcbded5
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-rerun-qwen397b.toml
@@ -0,0 +1,30 @@
+# GAIA rerun with max_turns=30: Qwen/Qwen3.5-397B-A17B-FP8
+[meta]
+name = "gaia-rerun-qwen397b"
+description = "GAIA rerun with 30 max turns on Qwen/Qwen3.5-397B-A17B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 1
+output_dir = "results/gaia-rerun-qwen397b/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-397B-A17B-FP8"
+engine = "vllm"
+num_gpus = 8
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["think", "calculator", "code_interpreter", "web_search", "file_read"]
diff --git a/src/openjarvis/evals/configs/livecodebench-kimi-k25.toml b/src/openjarvis/evals/configs/livecodebench-kimi-k25.toml
new file mode 100644
index 00000000..655949b5
--- /dev/null
+++ b/src/openjarvis/evals/configs/livecodebench-kimi-k25.toml
@@ -0,0 +1,39 @@
+# LiveCodeBench eval: Kimi-K2.5 (Ollama, IQ4_XS GGUF)
+# Competitive programming from LeetCode, AtCoder, CodeForces
+# Pull model: ollama pull hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS
+#
+# WARNING: Kimi-K2.5 is a 1T-param MoE (32B active). ALL GGUF quants are
+# sharded (smallest is 240 GB), and Ollama does not support sharded GGUFs
+# yet (see github.com/ollama/ollama/issues/5245). This config is a
+# placeholder -- use vLLM or wait for Ollama sharded GGUF support.
+# The IQ4_XS GGUF is 547 GB and would require 7x H100 80GB GPUs.
+
+[meta]
+name = "livecodebench-kimi-k25"
+description = "LiveCodeBench on Kimi-K2.5 IQ4_XS via Ollama"
+
+[defaults]
+temperature = 0.0
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/kimi-k25-ollama/livecodebench/"
+seed = 42
+
+[[models]]
+name = "hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS"
+engine = "ollama"
+param_count_b = 1000.0
+active_params_b = 32.0
+num_gpus = 7
+
+[[benchmarks]]
+name = "livecodebench"
+backend = "jarvis-direct"
+max_samples = 20
diff --git a/src/openjarvis/evals/configs/livecodebench-minimax-m25.toml b/src/openjarvis/evals/configs/livecodebench-minimax-m25.toml
new file mode 100644
index 00000000..73cf2930
--- /dev/null
+++ b/src/openjarvis/evals/configs/livecodebench-minimax-m25.toml
@@ -0,0 +1,36 @@
+# LiveCodeBench eval: MiniMax-M2.5 (Ollama, UD-TQ1_0 GGUF ~55 GB, GPUs 2-3)
+# Competitive programming from LeetCode, AtCoder, CodeForces
+# Pull model: ollama pull hf.co/unsloth/MiniMax-M2.5-GGUF:latest
+# NOTE: Q4_K_M and higher quants are sharded GGUFs which Ollama does not
+# support yet (see github.com/ollama/ollama/issues/5245). The "latest" tag
+# pulls the largest non-sharded quant: UD-TQ1_0 (1-bit, ~55 GB).
+
+[meta]
+name = "livecodebench-minimax-m25"
+description = "LiveCodeBench on MiniMax-M2.5 UD-TQ1_0 via Ollama"
+
+[defaults]
+temperature = 0.0
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/minimax-m25-ollama/livecodebench/"
+seed = 42
+
+[[models]]
+name = "hf.co/unsloth/MiniMax-M2.5-GGUF:latest"
+engine = "ollama"
+param_count_b = 229.0
+active_params_b = 45.6
+num_gpus = 2
+
+[[benchmarks]]
+name = "livecodebench"
+backend = "jarvis-direct"
+max_samples = 20
diff --git a/src/openjarvis/evals/configs/livecodebench-nemotron-super.toml b/src/openjarvis/evals/configs/livecodebench-nemotron-super.toml
new file mode 100644
index 00000000..30b8697b
--- /dev/null
+++ b/src/openjarvis/evals/configs/livecodebench-nemotron-super.toml
@@ -0,0 +1,28 @@
+# Livecodebench: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8
+[meta]
+name = "livecodebench-nemotron-super"
+description = "livecodebench on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
+
+[defaults]
+temperature = 0.0
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/nemotron-super/livecodebench/"
+seed = 42
+
+[[models]]
+name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
+engine = "vllm"
+num_gpus = 2
+
+[[benchmarks]]
+name = "livecodebench"
+backend = "jarvis-direct"
+max_samples = 20
diff --git a/src/openjarvis/evals/configs/livecodebench-qwen-122b.toml b/src/openjarvis/evals/configs/livecodebench-qwen-122b.toml
new file mode 100644
index 00000000..3ac0c749
--- /dev/null
+++ b/src/openjarvis/evals/configs/livecodebench-qwen-122b.toml
@@ -0,0 +1,28 @@
+# Livecodebench: Qwen/Qwen3.5-122B-A10B-FP8
+[meta]
+name = "livecodebench-qwen-122b"
+description = "livecodebench on Qwen/Qwen3.5-122B-A10B-FP8"
+
+[defaults]
+temperature = 0.0
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-122b/livecodebench/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-122B-A10B-FP8"
+engine = "vllm"
+num_gpus = 2
+
+[[benchmarks]]
+name = "livecodebench"
+backend = "jarvis-direct"
+max_samples = 20
diff --git a/src/openjarvis/evals/configs/livecodebench-qwen-35b.toml b/src/openjarvis/evals/configs/livecodebench-qwen-35b.toml
new file mode 100644
index 00000000..281a5e84
--- /dev/null
+++ b/src/openjarvis/evals/configs/livecodebench-qwen-35b.toml
@@ -0,0 +1,28 @@
+# Livecodebench: Qwen/Qwen3.5-35B-A3B-FP8
+[meta]
+name = "livecodebench-qwen-35b"
+description = "livecodebench on Qwen/Qwen3.5-35B-A3B-FP8"
+
+[defaults]
+temperature = 0.0
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-35b/livecodebench/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-35B-A3B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "livecodebench"
+backend = "jarvis-direct"
+max_samples = 20
diff --git a/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml b/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml
new file mode 100644
index 00000000..a398c7d0
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml
@@ -0,0 +1,31 @@
+# LiveResearchBench: Gemma-4-26B-A4B-it (vLLM, 1 GPU)
+[meta]
+name = "liveresearch-gemma4-26b"
+description = "LiveResearchBench on google/gemma-4-26B-A4B-it"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/gemma4-26b/liveresearch/"
+seed = 42
+
+[[models]]
+name = "google/gemma-4-26B-A4B-it"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-nemotron-super.toml b/src/openjarvis/evals/configs/liveresearch-nemotron-super.toml
new file mode 100644
index 00000000..f14956fa
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-nemotron-super.toml
@@ -0,0 +1,30 @@
+# Liveresearch: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8
+[meta]
+name = "liveresearch-nemotron-super"
+description = "liveresearch on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/nemotron-super/liveresearch/"
+seed = 42
+
+[[models]]
+name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
+engine = "vllm"
+num_gpus = 2
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+max_samples = 50
+agent = "monitor_operative"
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-122b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-122b.toml
new file mode 100644
index 00000000..0e316fa2
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-qwen-122b.toml
@@ -0,0 +1,30 @@
+# Liveresearch: Qwen/Qwen3.5-122B-A10B-FP8
+[meta]
+name = "liveresearch-qwen-122b"
+description = "liveresearch on Qwen/Qwen3.5-122B-A10B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-122b/liveresearch/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-122B-A10B-FP8"
+engine = "vllm"
+num_gpus = 2
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+max_samples = 50
+agent = "monitor_operative"
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml
new file mode 100644
index 00000000..d0beff38
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml
@@ -0,0 +1,31 @@
+# LiveResearchBench: Qwen3.5-2B (vLLM, 1 GPU)
+[meta]
+name = "liveresearch-qwen-2b"
+description = "LiveResearchBench on Qwen/Qwen3.5-2B"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-2b/liveresearch/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-2B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-35b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-35b.toml
new file mode 100644
index 00000000..483229ee
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-qwen-35b.toml
@@ -0,0 +1,30 @@
+# Liveresearch: Qwen/Qwen3.5-35B-A3B-FP8
+[meta]
+name = "liveresearch-qwen-35b"
+description = "liveresearch on Qwen/Qwen3.5-35B-A3B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-35b/liveresearch/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-35B-A3B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+max_samples = 50
+agent = "monitor_operative"
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml
new file mode 100644
index 00000000..002e02ae
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml
@@ -0,0 +1,31 @@
+# LiveResearchBench: Qwen3.5-9B (vLLM, 1 GPU)
+[meta]
+name = "liveresearch-qwen-9b"
+description = "LiveResearchBench on Qwen/Qwen3.5-9B"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-9b/liveresearch/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-9B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/pinchbench-kimi-k25.toml b/src/openjarvis/evals/configs/pinchbench-kimi-k25.toml
new file mode 100644
index 00000000..e5ef8667
--- /dev/null
+++ b/src/openjarvis/evals/configs/pinchbench-kimi-k25.toml
@@ -0,0 +1,44 @@
+# PinchBench eval: Kimi-K2.5 (Ollama, IQ4_XS GGUF)
+# Agent: native_openhands -- all PinchBench-required tools enabled
+# Pull model: ollama pull hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS
+#
+# WARNING: Kimi-K2.5 is a 1T-param MoE (32B active). ALL GGUF quants are
+# sharded (smallest is 240 GB), and Ollama does not support sharded GGUFs
+# yet (see github.com/ollama/ollama/issues/5245). This config is a
+# placeholder -- use vLLM or wait for Ollama sharded GGUF support.
+# The IQ4_XS GGUF is 547 GB and would require 7x H100 80GB GPUs.
+
+[meta]
+name = "pinchbench-kimi-k25"
+description = "PinchBench on Kimi-K2.5 IQ4_XS via Ollama"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "claude-opus-4-5"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/kimi-k25-ollama/pinchbench/"
+seed = 42
+
+[[models]]
+name = "hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS"
+engine = "ollama"
+param_count_b = 1000.0
+active_params_b = 32.0
+num_gpus = 7
+
+[[benchmarks]]
+name = "pinchbench"
+backend = "jarvis-agent"
+agent = "native_openhands"
+tools = [
+ "think", "file_read", "file_write", "web_search", "shell_exec",
+ "code_interpreter", "browser_navigate", "image_generate",
+ "calculator", "http_request", "pdf_extract",
+]
diff --git a/src/openjarvis/evals/configs/pinchbench-minimax-m25.toml b/src/openjarvis/evals/configs/pinchbench-minimax-m25.toml
new file mode 100644
index 00000000..e9662b3a
--- /dev/null
+++ b/src/openjarvis/evals/configs/pinchbench-minimax-m25.toml
@@ -0,0 +1,41 @@
+# PinchBench eval: MiniMax-M2.5 (Ollama, UD-TQ1_0 GGUF ~55 GB, GPUs 2-3)
+# Agent: native_openhands -- all PinchBench-required tools enabled
+# Pull model: ollama pull hf.co/unsloth/MiniMax-M2.5-GGUF:latest
+# NOTE: Q4_K_M and higher quants are sharded GGUFs which Ollama does not
+# support yet (see github.com/ollama/ollama/issues/5245). The "latest" tag
+# pulls the largest non-sharded quant: UD-TQ1_0 (1-bit, ~55 GB).
+
+[meta]
+name = "pinchbench-minimax-m25"
+description = "PinchBench on MiniMax-M2.5 UD-TQ1_0 via Ollama"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "claude-opus-4-5"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/minimax-m25-ollama/pinchbench/"
+seed = 42
+
+[[models]]
+name = "hf.co/unsloth/MiniMax-M2.5-GGUF:latest"
+engine = "ollama"
+param_count_b = 229.0
+active_params_b = 45.6
+num_gpus = 2
+
+[[benchmarks]]
+name = "pinchbench"
+backend = "jarvis-agent"
+agent = "native_openhands"
+tools = [
+ "think", "file_read", "file_write", "web_search", "shell_exec",
+ "code_interpreter", "browser_navigate", "image_generate",
+ "calculator", "http_request", "pdf_extract",
+]
diff --git a/src/openjarvis/evals/configs/pinchbench-nemotron-super.toml b/src/openjarvis/evals/configs/pinchbench-nemotron-super.toml
new file mode 100644
index 00000000..c66090ea
--- /dev/null
+++ b/src/openjarvis/evals/configs/pinchbench-nemotron-super.toml
@@ -0,0 +1,34 @@
+# Pinchbench: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8
+[meta]
+name = "pinchbench-nemotron-super"
+description = "pinchbench on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/nemotron-super/pinchbench/"
+seed = 42
+
+[[models]]
+name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
+engine = "vllm"
+num_gpus = 2
+
+[[benchmarks]]
+name = "pinchbench"
+backend = "jarvis-agent"
+max_samples = 23
+agent = "native_openhands"
+tools = [
+ "think", "file_read", "file_write", "web_search", "shell_exec",
+ "code_interpreter", "browser_navigate", "image_generate",
+ "calculator", "http_request", "pdf_extract",
+]
diff --git a/src/openjarvis/evals/configs/pinchbench-qwen-122b.toml b/src/openjarvis/evals/configs/pinchbench-qwen-122b.toml
new file mode 100644
index 00000000..5a91cc93
--- /dev/null
+++ b/src/openjarvis/evals/configs/pinchbench-qwen-122b.toml
@@ -0,0 +1,34 @@
+# Pinchbench: Qwen/Qwen3.5-122B-A10B-FP8
+[meta]
+name = "pinchbench-qwen-122b"
+description = "pinchbench on Qwen/Qwen3.5-122B-A10B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-122b/pinchbench/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-122B-A10B-FP8"
+engine = "vllm"
+num_gpus = 2
+
+[[benchmarks]]
+name = "pinchbench"
+backend = "jarvis-agent"
+max_samples = 23
+agent = "native_openhands"
+tools = [
+ "think", "file_read", "file_write", "web_search", "shell_exec",
+ "code_interpreter", "browser_navigate", "image_generate",
+ "calculator", "http_request", "pdf_extract",
+]
diff --git a/src/openjarvis/evals/configs/pinchbench-qwen-35b.toml b/src/openjarvis/evals/configs/pinchbench-qwen-35b.toml
new file mode 100644
index 00000000..1f2188ee
--- /dev/null
+++ b/src/openjarvis/evals/configs/pinchbench-qwen-35b.toml
@@ -0,0 +1,34 @@
+# Pinchbench: Qwen/Qwen3.5-35B-A3B-FP8
+[meta]
+name = "pinchbench-qwen-35b"
+description = "pinchbench on Qwen/Qwen3.5-35B-A3B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 8192
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-35b/pinchbench/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-35B-A3B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "pinchbench"
+backend = "jarvis-agent"
+max_samples = 23
+agent = "native_openhands"
+tools = [
+ "think", "file_read", "file_write", "web_search", "shell_exec",
+ "code_interpreter", "browser_navigate", "image_generate",
+ "calculator", "http_request", "pdf_extract",
+]
diff --git a/src/openjarvis/evals/configs/taubench-kimi-k25.toml b/src/openjarvis/evals/configs/taubench-kimi-k25.toml
new file mode 100644
index 00000000..639f3f05
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-kimi-k25.toml
@@ -0,0 +1,40 @@
+# TauBench V2 eval: Kimi-K2.5 (Ollama, IQ4_XS GGUF)
+# Multi-turn customer service benchmark -- airline + retail splits
+# Pull model: ollama pull hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS
+#
+# WARNING: Kimi-K2.5 is a 1T-param MoE (32B active). ALL GGUF quants are
+# sharded (smallest is 240 GB), and Ollama does not support sharded GGUFs
+# yet (see github.com/ollama/ollama/issues/5245). This config is a
+# placeholder -- use vLLM or wait for Ollama sharded GGUF support.
+# The IQ4_XS GGUF is 547 GB and would require 7x H100 80GB GPUs.
+
+[meta]
+name = "taubench-kimi-k25"
+description = "TauBench V2 on Kimi-K2.5 IQ4_XS via Ollama"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/kimi-k25-ollama/taubench/"
+seed = 42
+
+[[models]]
+name = "hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS"
+engine = "ollama"
+param_count_b = 1000.0
+active_params_b = 32.0
+num_gpus = 7
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "airline,retail"
diff --git a/src/openjarvis/evals/configs/taubench-minimax-m25.toml b/src/openjarvis/evals/configs/taubench-minimax-m25.toml
new file mode 100644
index 00000000..c962efd8
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-minimax-m25.toml
@@ -0,0 +1,37 @@
+# TauBench V2 eval: MiniMax-M2.5 (Ollama, UD-TQ1_0 GGUF ~55 GB, GPUs 2-3)
+# Multi-turn customer service benchmark -- airline + retail splits
+# Pull model: ollama pull hf.co/unsloth/MiniMax-M2.5-GGUF:latest
+# NOTE: Q4_K_M and higher quants are sharded GGUFs which Ollama does not
+# support yet (see github.com/ollama/ollama/issues/5245). The "latest" tag
+# pulls the largest non-sharded quant: UD-TQ1_0 (1-bit, ~55 GB).
+
+[meta]
+name = "taubench-minimax-m25"
+description = "TauBench V2 on MiniMax-M2.5 UD-TQ1_0 via Ollama"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/minimax-m25-ollama/taubench/"
+seed = 42
+
+[[models]]
+name = "hf.co/unsloth/MiniMax-M2.5-GGUF:latest"
+engine = "ollama"
+param_count_b = 229.0
+active_params_b = 45.6
+num_gpus = 2
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "airline,retail"
diff --git a/src/openjarvis/evals/configs/taubench-nemotron-super.toml b/src/openjarvis/evals/configs/taubench-nemotron-super.toml
new file mode 100644
index 00000000..f7f5304b
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-nemotron-super.toml
@@ -0,0 +1,29 @@
+# Taubench: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8
+[meta]
+name = "taubench-nemotron-super"
+description = "taubench on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/nemotron-super/taubench/"
+seed = 42
+
+[[models]]
+name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
+engine = "vllm"
+num_gpus = 2
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "airline,retail"
diff --git a/src/openjarvis/evals/configs/taubench-qwen-122b.toml b/src/openjarvis/evals/configs/taubench-qwen-122b.toml
new file mode 100644
index 00000000..a0f4329a
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-qwen-122b.toml
@@ -0,0 +1,29 @@
+# Taubench: Qwen/Qwen3.5-122B-A10B-FP8
+[meta]
+name = "taubench-qwen-122b"
+description = "taubench on Qwen/Qwen3.5-122B-A10B-FP8"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-122b/taubench/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-122B-A10B-FP8"
+engine = "vllm"
+num_gpus = 2
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "airline,retail"
diff --git a/src/openjarvis/evals/configs/taubench-qwen-35b.toml b/src/openjarvis/evals/configs/taubench-qwen-35b.toml
new file mode 100644
index 00000000..d902364b
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-qwen-35b.toml
@@ -0,0 +1,29 @@
+# Taubench: Qwen/Qwen3.5-35B-A3B-FP8
+[meta]
+name = "taubench-qwen-35b"
+description = "taubench on Qwen/Qwen3.5-35B-A3B-FP8"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-35b/taubench/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-35B-A3B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "airline,retail"
diff --git a/src/openjarvis/evals/configs/taubench-telecom-claude.toml b/src/openjarvis/evals/configs/taubench-telecom-claude.toml
new file mode 100644
index 00000000..2710b225
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-telecom-claude.toml
@@ -0,0 +1,27 @@
+# TauBench V2 Telecom: Claude Opus 4.6
+[meta]
+name = "taubench-telecom-claude"
+description = "TauBench V2 telecom on claude-opus-4-6 (test split, pass^3)"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/taubench-telecom-claude/"
+seed = 42
+
+[[models]]
+name = "claude-opus-4-6"
+engine = "cloud"
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+split = "telecom"
diff --git a/src/openjarvis/evals/configs/taubench-telecom-gemma4-26b.toml b/src/openjarvis/evals/configs/taubench-telecom-gemma4-26b.toml
new file mode 100644
index 00000000..380ab271
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-telecom-gemma4-26b.toml
@@ -0,0 +1,29 @@
+# TauBench V2 Telecom: Gemma-4-26B-A4B-it (vLLM, 1 GPU)
+[meta]
+name = "taubench-telecom-gemma4-26b"
+description = "TauBench V2 Telecom domain on google/gemma-4-26B-A4B-it"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/gemma4-26b/taubench-telecom/"
+seed = 42
+
+[[models]]
+name = "google/gemma-4-26B-A4B-it"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "telecom"
diff --git a/src/openjarvis/evals/configs/taubench-telecom-gpt54.toml b/src/openjarvis/evals/configs/taubench-telecom-gpt54.toml
new file mode 100644
index 00000000..1dbb3f3a
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-telecom-gpt54.toml
@@ -0,0 +1,27 @@
+# TauBench V2 Telecom: GPT-5.4
+[meta]
+name = "taubench-telecom-gpt54"
+description = "TauBench V2 telecom on gpt-5.4 (test split, pass^3)"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/taubench-telecom-gpt54/"
+seed = 42
+
+[[models]]
+name = "gpt-5.4-2026-03-05"
+engine = "cloud"
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+split = "telecom"
diff --git a/src/openjarvis/evals/configs/taubench-telecom-nemotron-super.toml b/src/openjarvis/evals/configs/taubench-telecom-nemotron-super.toml
new file mode 100644
index 00000000..26cffbd2
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-telecom-nemotron-super.toml
@@ -0,0 +1,29 @@
+# Taubench: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8
+[meta]
+name = "taubench-telecom-nemotron-super"
+description = "taubench on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/nemotron-super/taubench-telecom/"
+seed = 42
+
+[[models]]
+name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
+engine = "vllm"
+num_gpus = 2
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "telecom"
diff --git a/src/openjarvis/evals/configs/taubench-telecom-nemotron.toml b/src/openjarvis/evals/configs/taubench-telecom-nemotron.toml
new file mode 100644
index 00000000..d71b37a2
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-telecom-nemotron.toml
@@ -0,0 +1,28 @@
+# TauBench V2 Telecom: Nemotron-3-Super (SGLang port 8001)
+[meta]
+name = "taubench-telecom-nemotron"
+description = "TauBench V2 telecom on Nemotron-3-Super (test split, pass^3)"
+
+[defaults]
+temperature = 1.0
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/taubench-telecom-nemotron/"
+seed = 42
+
+[[models]]
+name = "nemotron"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+split = "telecom"
diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen-122b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen-122b.toml
new file mode 100644
index 00000000..849fb30e
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-telecom-qwen-122b.toml
@@ -0,0 +1,29 @@
+# Taubench: Qwen/Qwen3.5-122B-A10B-FP8
+[meta]
+name = "taubench-telecom-qwen-122b"
+description = "taubench on Qwen/Qwen3.5-122B-A10B-FP8"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-122b/taubench-telecom/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-122B-A10B-FP8"
+engine = "vllm"
+num_gpus = 2
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "telecom"
diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen-35b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen-35b.toml
new file mode 100644
index 00000000..ccf5c8f6
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-telecom-qwen-35b.toml
@@ -0,0 +1,29 @@
+# Taubench: Qwen/Qwen3.5-35B-A3B-FP8
+[meta]
+name = "taubench-telecom-qwen-35b"
+description = "taubench on Qwen/Qwen3.5-35B-A3B-FP8"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-35b/taubench-telecom/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-35B-A3B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "telecom"
diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen27b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen27b.toml
new file mode 100644
index 00000000..c4a2de6f
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-telecom-qwen27b.toml
@@ -0,0 +1,29 @@
+# TauBench V2 Telecom: Qwen3.5-27B-FP8 (vLLM, 1 GPU)
+[meta]
+name = "taubench-telecom-qwen27b"
+description = "TauBench V2 Telecom domain on Qwen/Qwen3.5-27B-FP8"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-27b/taubench-telecom/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-27B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "telecom"
diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml
new file mode 100644
index 00000000..d1fd6023
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-telecom-qwen2b.toml
@@ -0,0 +1,29 @@
+# TauBench V2 Telecom: Qwen3.5-2B (vLLM, 1 GPU)
+[meta]
+name = "taubench-telecom-qwen2b"
+description = "TauBench V2 Telecom domain on Qwen/Qwen3.5-2B"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-2b/taubench-telecom/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-2B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "telecom"
diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen9b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen9b.toml
new file mode 100644
index 00000000..85d8691e
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-telecom-qwen9b.toml
@@ -0,0 +1,29 @@
+# TauBench V2 Telecom: Qwen3.5-9B (vLLM, 1 GPU)
+[meta]
+name = "taubench-telecom-qwen9b"
+description = "TauBench V2 Telecom domain on Qwen/Qwen3.5-9B"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-9b/taubench-telecom/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-9B"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+max_samples = 20
+split = "telecom"
diff --git a/src/openjarvis/evals/configs/toolcall15-kimi-k25.toml b/src/openjarvis/evals/configs/toolcall15-kimi-k25.toml
new file mode 100644
index 00000000..2f0596bc
--- /dev/null
+++ b/src/openjarvis/evals/configs/toolcall15-kimi-k25.toml
@@ -0,0 +1,38 @@
+# ToolCall-15 eval: Kimi-K2.5 (Ollama, IQ4_XS GGUF)
+# Lightweight tool calling benchmark -- 15 scenarios, 5 categories
+# Pull model: ollama pull hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS
+#
+# WARNING: Kimi-K2.5 is a 1T-param MoE (32B active). ALL GGUF quants are
+# sharded (smallest is 240 GB), and Ollama does not support sharded GGUFs
+# yet (see github.com/ollama/ollama/issues/5245). This config is a
+# placeholder -- use vLLM or wait for Ollama sharded GGUF support.
+# The IQ4_XS GGUF is 547 GB and would require 7x H100 80GB GPUs.
+
+[meta]
+name = "toolcall15-kimi-k25"
+description = "ToolCall-15 on Kimi-K2.5 IQ4_XS via Ollama"
+
+[defaults]
+temperature = 0.0
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/kimi-k25-ollama/toolcall15/"
+seed = 42
+
+[[models]]
+name = "hf.co/unsloth/Kimi-K2.5-GGUF:IQ4_XS"
+engine = "ollama"
+param_count_b = 1000.0
+active_params_b = 32.0
+num_gpus = 7
+
+[[benchmarks]]
+name = "toolcall15"
+backend = "jarvis-direct"
diff --git a/src/openjarvis/evals/configs/toolcall15-minimax-m25.toml b/src/openjarvis/evals/configs/toolcall15-minimax-m25.toml
new file mode 100644
index 00000000..6d087419
--- /dev/null
+++ b/src/openjarvis/evals/configs/toolcall15-minimax-m25.toml
@@ -0,0 +1,35 @@
+# ToolCall-15 eval: MiniMax-M2.5 (Ollama, UD-TQ1_0 GGUF ~55 GB, GPUs 2-3)
+# Lightweight tool calling benchmark -- 15 scenarios, 5 categories
+# Pull model: ollama pull hf.co/unsloth/MiniMax-M2.5-GGUF:latest
+# NOTE: Q4_K_M and higher quants are sharded GGUFs which Ollama does not
+# support yet (see github.com/ollama/ollama/issues/5245). The "latest" tag
+# pulls the largest non-sharded quant: UD-TQ1_0 (1-bit, ~55 GB).
+
+[meta]
+name = "toolcall15-minimax-m25"
+description = "ToolCall-15 on MiniMax-M2.5 UD-TQ1_0 via Ollama"
+
+[defaults]
+temperature = 0.0
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/minimax-m25-ollama/toolcall15/"
+seed = 42
+
+[[models]]
+name = "hf.co/unsloth/MiniMax-M2.5-GGUF:latest"
+engine = "ollama"
+param_count_b = 229.0
+active_params_b = 45.6
+num_gpus = 2
+
+[[benchmarks]]
+name = "toolcall15"
+backend = "jarvis-direct"
diff --git a/src/openjarvis/evals/configs/toolcall15-nemotron-super.toml b/src/openjarvis/evals/configs/toolcall15-nemotron-super.toml
new file mode 100644
index 00000000..9b3fba0f
--- /dev/null
+++ b/src/openjarvis/evals/configs/toolcall15-nemotron-super.toml
@@ -0,0 +1,28 @@
+# Toolcall15: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8
+[meta]
+name = "toolcall15-nemotron-super"
+description = "toolcall15 on nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/nemotron-super/toolcall15/"
+seed = 42
+
+[[models]]
+name = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
+engine = "vllm"
+num_gpus = 2
+
+[[benchmarks]]
+name = "toolcall15"
+backend = "jarvis-direct"
+max_samples = 15
diff --git a/src/openjarvis/evals/configs/toolcall15-qwen-122b.toml b/src/openjarvis/evals/configs/toolcall15-qwen-122b.toml
new file mode 100644
index 00000000..b88527e4
--- /dev/null
+++ b/src/openjarvis/evals/configs/toolcall15-qwen-122b.toml
@@ -0,0 +1,28 @@
+# Toolcall15: Qwen/Qwen3.5-122B-A10B-FP8
+[meta]
+name = "toolcall15-qwen-122b"
+description = "toolcall15 on Qwen/Qwen3.5-122B-A10B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-122b/toolcall15/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-122B-A10B-FP8"
+engine = "vllm"
+num_gpus = 2
+
+[[benchmarks]]
+name = "toolcall15"
+backend = "jarvis-direct"
+max_samples = 15
diff --git a/src/openjarvis/evals/configs/toolcall15-qwen-35b.toml b/src/openjarvis/evals/configs/toolcall15-qwen-35b.toml
new file mode 100644
index 00000000..7dd7ccee
--- /dev/null
+++ b/src/openjarvis/evals/configs/toolcall15-qwen-35b.toml
@@ -0,0 +1,28 @@
+# Toolcall15: Qwen/Qwen3.5-35B-A3B-FP8
+[meta]
+name = "toolcall15-qwen-35b"
+description = "toolcall15 on Qwen/Qwen3.5-35B-A3B-FP8"
+
+[defaults]
+temperature = 0.6
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/neurips-2026/baselines/qwen-35b/toolcall15/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-35B-A3B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "toolcall15"
+backend = "jarvis-direct"
+max_samples = 15