Personal AI, On Personal Devices
Python
fix(evals): TerminalBench scoring + http_request panic + 20 new configs (#173)
Avanika Narayan897a81634304
diff --git a/src/openjarvis/evals/configs/livecodebench-gemma4-26b.toml b/src/openjarvis/evals/configs/livecodebench-gemma4-26b.tomlnew file mode 100644index 00000000..d427f01f--- /dev/null+++ b/src/openjarvis/evals/configs/livecodebench-gemma4-26b.toml@@ -0,0 +1,30 @@+# LiveCodeBench eval: Gemma-4-26B-A4B-it (vLLM, TP=4)+# Competitive programming from LeetCode, AtCoder, CodeForces++[meta]+name = "livecodebench-gemma4-26b"+description = "LiveCodeBench on google/gemma-4-26B-A4B-it (vLLM, TP=4)"++[defaults]+temperature = 0.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/gemma4-26b/livecodebench/"+seed = 42++[[models]]+name = "google/gemma-4-26B-A4B-it"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "livecodebench"+backend = "jarvis-direct"+max_samples = 20diff --git a/src/openjarvis/evals/configs/livecodebench-gemma4-e4b.toml b/src/openjarvis/evals/configs/livecodebench-gemma4-e4b.tomlnew file mode 100644index 00000000..ff6209fa--- /dev/null+++ b/src/openjarvis/evals/configs/livecodebench-gemma4-e4b.toml@@ -0,0 +1,30 @@+# LiveCodeBench eval: Gemma-4-E4B-it (vLLM, 1 GPU)+# Competitive programming from LeetCode, AtCoder, CodeForces++[meta]+name = "livecodebench-gemma4-e4b"+description = "LiveCodeBench on google/gemma-4-E4B-it (vLLM, 1 GPU)"++[defaults]+temperature = 0.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/gemma4-e4b/livecodebench/"+seed = 42++[[models]]+name = "google/gemma-4-E4B-it"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "livecodebench"+backend = "jarvis-direct"+max_samples = 20diff --git a/src/openjarvis/evals/configs/livecodebench-nemotron-nano.toml b/src/openjarvis/evals/configs/livecodebench-nemotron-nano.tomlnew file mode 100644index 00000000..6266f922--- /dev/null+++ b/src/openjarvis/evals/configs/livecodebench-nemotron-nano.toml@@ -0,0 +1,30 @@+# LiveCodeBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)+# Competitive programming from LeetCode, AtCoder, CodeForces++[meta]+name = "livecodebench-nemotron-nano"+description = "LiveCodeBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)"++[defaults]+temperature = 0.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/nemotron-nano/livecodebench/"+seed = 42++[[models]]+name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "livecodebench"+backend = "jarvis-direct"+max_samples = 20diff --git a/src/openjarvis/evals/configs/livecodebench-qwen-27b.toml b/src/openjarvis/evals/configs/livecodebench-qwen-27b.tomlnew file mode 100644index 00000000..357dcf49--- /dev/null+++ b/src/openjarvis/evals/configs/livecodebench-qwen-27b.toml@@ -0,0 +1,30 @@+# LiveCodeBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4)+# Competitive programming from LeetCode, AtCoder, CodeForces++[meta]+name = "livecodebench-qwen-27b"+description = "LiveCodeBench on Qwen/Qwen3.5-27B-FP8 (vLLM, TP=4)"++[defaults]+temperature = 0.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-27b/livecodebench/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-27B-FP8"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "livecodebench"+backend = "jarvis-direct"+max_samples = 20diff --git a/src/openjarvis/evals/configs/livecodebench-qwen-2b.toml b/src/openjarvis/evals/configs/livecodebench-qwen-2b.tomlnew file mode 100644index 00000000..e4d4236e--- /dev/null+++ b/src/openjarvis/evals/configs/livecodebench-qwen-2b.toml@@ -0,0 +1,30 @@+# LiveCodeBench eval: Qwen3.5-2B (vLLM, 1 GPU)+# Competitive programming from LeetCode, AtCoder, CodeForces++[meta]+name = "livecodebench-qwen-2b"+description = "LiveCodeBench on Qwen/Qwen3.5-2B (vLLM, 1 GPU)"++[defaults]+temperature = 0.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-2b/livecodebench/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-2B"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "livecodebench"+backend = "jarvis-direct"+max_samples = 20diff --git a/src/openjarvis/evals/configs/pinchbench-gemma4-26b.toml b/src/openjarvis/evals/configs/pinchbench-gemma4-26b.tomlnew file mode 100644index 00000000..28787489--- /dev/null+++ b/src/openjarvis/evals/configs/pinchbench-gemma4-26b.toml@@ -0,0 +1,35 @@+# PinchBench eval: Gemma-4-26B-A4B-it (vLLM, TP=4)+# Agent: native_openhands — all PinchBench-required tools enabled++[meta]+name = "pinchbench-gemma4-26b"+description = "PinchBench on google/gemma-4-26B-A4B-it (vLLM, TP=4)"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "claude-opus-4-5"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/gemma4-26b/pinchbench/"+seed = 42++[[models]]+name = "google/gemma-4-26B-A4B-it"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "pinchbench"+backend = "jarvis-agent"+agent = "native_openhands"+tools = [+ "think", "file_read", "file_write", "web_search", "shell_exec",+ "code_interpreter", "browser_navigate", "image_generate",+ "calculator", "http_request", "pdf_extract",+]diff --git a/src/openjarvis/evals/configs/pinchbench-gemma4-e4b.toml b/src/openjarvis/evals/configs/pinchbench-gemma4-e4b.tomlnew file mode 100644index 00000000..bbe9ba30--- /dev/null+++ b/src/openjarvis/evals/configs/pinchbench-gemma4-e4b.toml@@ -0,0 +1,35 @@+# PinchBench eval: Gemma-4-E4B-it (vLLM, 1 GPU)+# Agent: native_openhands — all PinchBench-required tools enabled++[meta]+name = "pinchbench-gemma4-e4b"+description = "PinchBench on google/gemma-4-E4B-it (vLLM, 1 GPU)"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "claude-opus-4-5"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/gemma4-e4b/pinchbench/"+seed = 42++[[models]]+name = "google/gemma-4-E4B-it"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "pinchbench"+backend = "jarvis-agent"+agent = "native_openhands"+tools = [+ "think", "file_read", "file_write", "web_search", "shell_exec",+ "code_interpreter", "browser_navigate", "image_generate",+ "calculator", "http_request", "pdf_extract",+]diff --git a/src/openjarvis/evals/configs/pinchbench-nemotron-nano.toml b/src/openjarvis/evals/configs/pinchbench-nemotron-nano.tomlnew file mode 100644index 00000000..5ddadce5--- /dev/null+++ b/src/openjarvis/evals/configs/pinchbench-nemotron-nano.toml@@ -0,0 +1,35 @@+# PinchBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)+# Agent: native_openhands — all PinchBench-required tools enabled++[meta]+name = "pinchbench-nemotron-nano"+description = "PinchBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "claude-opus-4-5"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/nemotron-nano/pinchbench/"+seed = 42++[[models]]+name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "pinchbench"+backend = "jarvis-agent"+agent = "native_openhands"+tools = [+ "think", "file_read", "file_write", "web_search", "shell_exec",+ "code_interpreter", "browser_navigate", "image_generate",+ "calculator", "http_request", "pdf_extract",+]diff --git a/src/openjarvis/evals/configs/pinchbench-qwen-27b.toml b/src/openjarvis/evals/configs/pinchbench-qwen-27b.tomlnew file mode 100644index 00000000..f687e63d--- /dev/null+++ b/src/openjarvis/evals/configs/pinchbench-qwen-27b.toml@@ -0,0 +1,35 @@+# PinchBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4)+# Agent: native_openhands — all PinchBench-required tools enabled++[meta]+name = "pinchbench-qwen-27b"+description = "PinchBench on Qwen/Qwen3.5-27B-FP8 (vLLM, TP=4)"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "claude-opus-4-5"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-27b/pinchbench/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-27B-FP8"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "pinchbench"+backend = "jarvis-agent"+agent = "native_openhands"+tools = [+ "think", "file_read", "file_write", "web_search", "shell_exec",+ "code_interpreter", "browser_navigate", "image_generate",+ "calculator", "http_request", "pdf_extract",+]diff --git a/src/openjarvis/evals/configs/pinchbench-qwen-2b.toml b/src/openjarvis/evals/configs/pinchbench-qwen-2b.tomlnew file mode 100644index 00000000..e76e56a4--- /dev/null+++ b/src/openjarvis/evals/configs/pinchbench-qwen-2b.toml@@ -0,0 +1,35 @@+# PinchBench eval: Qwen3.5-2B (vLLM, 1 GPU)+# Agent: native_openhands — all PinchBench-required tools enabled++[meta]+name = "pinchbench-qwen-2b"+description = "PinchBench on Qwen/Qwen3.5-2B (vLLM, 1 GPU)"++[defaults]+temperature = 0.6+max_tokens = 8192++[judge]+model = "claude-opus-4-5"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-2b/pinchbench/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-2B"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "pinchbench"+backend = "jarvis-agent"+agent = "native_openhands"+tools = [+ "think", "file_read", "file_write", "web_search", "shell_exec",+ "code_interpreter", "browser_navigate", "image_generate",+ "calculator", "http_request", "pdf_extract",+]diff --git a/src/openjarvis/evals/configs/taubench-gemma4-26b.toml b/src/openjarvis/evals/configs/taubench-gemma4-26b.tomlnew file mode 100644index 00000000..27fda44b--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-gemma4-26b.toml@@ -0,0 +1,31 @@+# TauBench V2 eval: Gemma-4-26B-A4B-it (vLLM, TP=4)+# Multi-turn customer service benchmark — airline + retail splits++[meta]+name = "taubench-gemma4-26b"+description = "TauBench V2 on google/gemma-4-26B-A4B-it (vLLM, TP=4)"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/gemma4-26b/taubench/"+seed = 42++[[models]]+name = "google/gemma-4-26B-A4B-it"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+max_samples = 20+split = "airline,retail"diff --git a/src/openjarvis/evals/configs/taubench-gemma4-e4b.toml b/src/openjarvis/evals/configs/taubench-gemma4-e4b.tomlnew file mode 100644index 00000000..8dac58dd--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-gemma4-e4b.toml@@ -0,0 +1,31 @@+# TauBench V2 eval: Gemma-4-E4B-it (vLLM, 1 GPU)+# Multi-turn customer service benchmark — airline + retail splits++[meta]+name = "taubench-gemma4-e4b"+description = "TauBench V2 on google/gemma-4-E4B-it (vLLM, 1 GPU)"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/gemma4-e4b/taubench/"+seed = 42++[[models]]+name = "google/gemma-4-E4B-it"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+max_samples = 20+split = "airline,retail"diff --git a/src/openjarvis/evals/configs/taubench-nemotron-nano.toml b/src/openjarvis/evals/configs/taubench-nemotron-nano.tomlnew file mode 100644index 00000000..7dff2f08--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-nemotron-nano.toml@@ -0,0 +1,31 @@+# TauBench V2 eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)+# Multi-turn customer service benchmark — airline + retail splits++[meta]+name = "taubench-nemotron-nano"+description = "TauBench V2 on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/nemotron-nano/taubench/"+seed = 42++[[models]]+name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+max_samples = 20+split = "airline,retail"diff --git a/src/openjarvis/evals/configs/taubench-qwen-27b.toml b/src/openjarvis/evals/configs/taubench-qwen-27b.tomlnew file mode 100644index 00000000..13ce0c65--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-qwen-27b.toml@@ -0,0 +1,31 @@+# TauBench V2 eval: Qwen3.5-27B-FP8 (vLLM, TP=4)+# Multi-turn customer service benchmark — airline + retail splits++[meta]+name = "taubench-qwen-27b"+description = "TauBench V2 on Qwen/Qwen3.5-27B-FP8 (vLLM, TP=4)"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-27b/taubench/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-27B-FP8"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+max_samples = 20+split = "airline,retail"diff --git a/src/openjarvis/evals/configs/taubench-qwen-2b.toml b/src/openjarvis/evals/configs/taubench-qwen-2b.tomlnew file mode 100644index 00000000..e9735ff6--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-qwen-2b.toml@@ -0,0 +1,31 @@+# TauBench V2 eval: Qwen3.5-2B (vLLM, 1 GPU)+# Multi-turn customer service benchmark — airline + retail splits++[meta]+name = "taubench-qwen-2b"+description = "TauBench V2 on Qwen/Qwen3.5-2B (vLLM, 1 GPU)"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-2b/taubench/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-2B"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+max_samples = 20+split = "airline,retail"diff --git a/src/openjarvis/evals/configs/toolcall15-gemma4-26b.toml b/src/openjarvis/evals/configs/toolcall15-gemma4-26b.tomlnew file mode 100644index 00000000..12470e25--- /dev/null+++ b/src/openjarvis/evals/configs/toolcall15-gemma4-26b.toml@@ -0,0 +1,29 @@+# ToolCall-15 eval: Gemma-4-26B-A4B-it (vLLM, TP=4)+# Lightweight tool calling benchmark — 15 scenarios, 5 categories++[meta]+name = "toolcall15-gemma4-26b"+description = "ToolCall-15 on google/gemma-4-26B-A4B-it (temperature=0)"++[defaults]+temperature = 0.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/gemma4-26b/toolcall15/"+seed = 42++[[models]]+name = "google/gemma-4-26B-A4B-it"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "toolcall15"+backend = "jarvis-direct"diff --git a/src/openjarvis/evals/configs/toolcall15-gemma4-e4b.toml b/src/openjarvis/evals/configs/toolcall15-gemma4-e4b.tomlnew file mode 100644index 00000000..231bed02--- /dev/null+++ b/src/openjarvis/evals/configs/toolcall15-gemma4-e4b.toml@@ -0,0 +1,29 @@+# ToolCall-15 eval: Gemma-4-E4B-it (vLLM, 1 GPU)+# Lightweight tool calling benchmark — 15 scenarios, 5 categories++[meta]+name = "toolcall15-gemma4-e4b"+description = "ToolCall-15 on google/gemma-4-E4B-it (temperature=0)"++[defaults]+temperature = 0.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/gemma4-e4b/toolcall15/"+seed = 42++[[models]]+name = "google/gemma-4-E4B-it"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "toolcall15"+backend = "jarvis-direct"diff --git a/src/openjarvis/evals/configs/toolcall15-nemotron-nano.toml b/src/openjarvis/evals/configs/toolcall15-nemotron-nano.tomlnew file mode 100644index 00000000..453f30e9--- /dev/null+++ b/src/openjarvis/evals/configs/toolcall15-nemotron-nano.toml@@ -0,0 +1,29 @@+# ToolCall-15 eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)+# Lightweight tool calling benchmark — 15 scenarios, 5 categories++[meta]+name = "toolcall15-nemotron-nano"+description = "ToolCall-15 on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (temperature=0)"++[defaults]+temperature = 0.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/nemotron-nano/toolcall15/"+seed = 42++[[models]]+name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "toolcall15"+backend = "jarvis-direct"diff --git a/src/openjarvis/evals/configs/toolcall15-qwen-27b.toml b/src/openjarvis/evals/configs/toolcall15-qwen-27b.tomlnew file mode 100644index 00000000..56879653--- /dev/null+++ b/src/openjarvis/evals/configs/toolcall15-qwen-27b.toml@@ -0,0 +1,29 @@+# ToolCall-15 eval: Qwen3.5-27B-FP8 (vLLM, TP=4)+# Lightweight tool calling benchmark — 15 scenarios, 5 categories++[meta]+name = "toolcall15-qwen-27b"+description = "ToolCall-15 on Qwen/Qwen3.5-27B-FP8 (temperature=0)"++[defaults]+temperature = 0.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-27b/toolcall15/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-27B-FP8"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "toolcall15"+backend = "jarvis-direct"diff --git a/src/openjarvis/evals/configs/toolcall15-qwen-2b.toml b/src/openjarvis/evals/configs/toolcall15-qwen-2b.tomlnew file mode 100644index 00000000..4d40fe42--- /dev/null+++ b/src/openjarvis/evals/configs/toolcall15-qwen-2b.toml@@ -0,0 +1,29 @@+# ToolCall-15 eval: Qwen3.5-2B (vLLM, 1 GPU)+# Lightweight tool calling benchmark — 15 scenarios, 5 categories++[meta]+name = "toolcall15-qwen-2b"+description = "ToolCall-15 on Qwen/Qwen3.5-2B (temperature=0)"++[defaults]+temperature = 0.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/neurips-2026/baselines/qwen-2b/toolcall15/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-2B"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "toolcall15"+backend = "jarvis-direct"