Personal AI, On Personal Devices
Python
fix: rename DeepResearchBench references in docs, configs, and tests
Jon Saad-Falcondb4b031c452f
diff --git a/docs/experiments/neurips-2026-plan.md b/docs/experiments/neurips-2026-plan.mdindex 4539b4bf..ccd614f4 100644--- a/docs/experiments/neurips-2026-plan.md+++ b/docs/experiments/neurips-2026-plan.md@@ -117,7 +117,7 @@ Key comparisons:| terminalbench | TerminalBench | varies | 20 | Implemented || toolcall15 | ToolCall-15 | 15 | 15 (all) | TODO || livecodebench | LiveCodeBench | ~100 | 20 | TODO |-| liveresearch | LiveResearchBench | 100 | 10 | TODO |+| liveresearch | DeepResearchBench | 100 | 10 | TODO |## Metrics Captured Per Run- accuracy (benchmark-specific)@@ -138,7 +138,7 @@ Key comparisons:### Phase 1a: Implement missing benchmarks- [ ] ToolCall-15 integration- [ ] LiveCodeBench integration-- [ ] LiveResearchBench integration+- [ ] DeepResearchBench integration- [ ] Wire telemetry capture to all eval runs### Phase 1b: Run cloud baselines (no GPU needed)@@ -148,7 +148,7 @@ Key comparisons:TauBench Telecom (75%), GAIA (34.29%)- [x] Gemini 3.1 Pro — PinchBench (78.26%), TauBench A+R (58.33%),TauBench Telecom (77.5%), GAIA (47.06%)-- [ ] All 3 cloud baselines — ToolCall-15, LiveCodeBench, LiveResearchBench+- [ ] All 3 cloud baselines — ToolCall-15, LiveCodeBench, DeepResearchBench- [ ] All 3 cloud baselines — TerminalBench### Phase 1c: Run local models (GPU required)@@ -230,7 +230,7 @@ Training targets:- Qwen 35B: TauBench telecom + GAIA running- ToolCall-15 integration: TODO- LiveCodeBench integration: TODO-- LiveResearchBench integration: TODO+- DeepResearchBench integration: TODO- Telemetry wiring: TODO### Blockeddiff --git a/src/openjarvis/evals/configs/liveresearch-claude-opus.toml b/src/openjarvis/evals/configs/liveresearch-claude-opus.tomlindex 71d173e0..e5bc67d2 100644--- a/src/openjarvis/evals/configs/liveresearch-claude-opus.toml+++ b/src/openjarvis/evals/configs/liveresearch-claude-opus.toml@@ -1,7 +1,7 @@-# LiveResearchBench: claude-opus-4-6+# DeepResearchBench: claude-opus-4-6[meta]name = "liveresearch-claude-opus"-description = "LiveResearchBench deep research benchmark on claude-opus-4-6"+description = "DeepResearchBench deep research benchmark on claude-opus-4-6"[defaults]temperature = 0.6diff --git a/src/openjarvis/evals/configs/liveresearch-gemini-31-pro.toml b/src/openjarvis/evals/configs/liveresearch-gemini-31-pro.tomlnew file mode 100644index 00000000..533bffbe--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-gemini-31-pro.toml@@ -0,0 +1,29 @@+# liveresearch: gemini-3.1-pro-preview (cloud)+[meta]+name = "liveresearch-gemini-31-pro"+description = "liveresearch on gemini-3.1-pro-preview"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 8+output_dir = "results/neurips-2026/baselines/gemini-31-pro/liveresearch/"+seed = 42++[[models]]+name = "gemini-3.1-pro-preview"+engine = "cloud"++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+max_samples = 50+agent = "monitor_operative"+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml b/src/openjarvis/evals/configs/liveresearch-gemma4-26b.tomlindex a398c7d0..feaba829 100644--- a/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml+++ b/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml@@ -1,7 +1,7 @@-# LiveResearchBench: Gemma-4-26B-A4B-it (vLLM, 1 GPU)+# DeepResearchBench: Gemma-4-26B-A4B-it (vLLM, 1 GPU)[meta]name = "liveresearch-gemma4-26b"-description = "LiveResearchBench on google/gemma-4-26B-A4B-it"+description = "DeepResearchBench on google/gemma-4-26B-A4B-it"[defaults]temperature = 0.6diff --git a/src/openjarvis/evals/configs/liveresearch-gemma4-31b.toml b/src/openjarvis/evals/configs/liveresearch-gemma4-31b.tomlnew file mode 100644index 00000000..da337db1--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-gemma4-31b.toml@@ -0,0 +1,30 @@+# liveresearch: google/gemma-4-31B-it+[meta]+name = "liveresearch-gemma4-31b"+description = "liveresearch on google/gemma-4-31B-it"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 16+output_dir = "results/neurips-2026/baselines/gemma4-31b/liveresearch/"+seed = 42++[[models]]+name = "google/gemma-4-31B-it"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+max_samples = 50+agent = "monitor_operative"+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-gpt54.toml b/src/openjarvis/evals/configs/liveresearch-gpt54.tomlnew file mode 100644index 00000000..904f32e7--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-gpt54.toml@@ -0,0 +1,29 @@+# liveresearch: gpt-5.4-2026-03-05 (cloud)+[meta]+name = "liveresearch-gpt54"+description = "liveresearch on gpt-5.4-2026-03-05"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 8+output_dir = "results/neurips-2026/baselines/gpt54/liveresearch/"+seed = 42++[[models]]+name = "gpt-5.4-2026-03-05"+engine = "cloud"++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+max_samples = 50+agent = "monitor_operative"+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b-lora.toml b/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b-lora.tomlnew file mode 100644index 00000000..ad3816ca--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b-lora.toml@@ -0,0 +1,30 @@+# liveresearch: granite33-agent+[meta]+name = "liveresearch-granite-3.3-8b-lora"+description = "liveresearch on granite33-agent"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 16+output_dir = "results/neurips-2026/optimized-eval/track-a/granite-3.3-8b-lora-lora/liveresearch/"+seed = 42++[[models]]+name = "granite33-agent"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+max_samples = 50+agent = "monitor_operative"+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b.toml b/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b.tomlnew file mode 100644index 00000000..7a29c69c--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b.toml@@ -0,0 +1,30 @@+# liveresearch: ibm-granite/granite-3.3-8b-instruct+[meta]+name = "liveresearch-granite-3.3-8b"+description = "liveresearch on ibm-granite/granite-3.3-8b-instruct"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 16+output_dir = "results/neurips-2026/baselines/granite-3.3-8b/liveresearch/"+seed = 42++[[models]]+name = "ibm-granite/granite-3.3-8b-instruct"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+max_samples = 50+agent = "monitor_operative"+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-granite-4.0-h-small.toml b/src/openjarvis/evals/configs/liveresearch-granite-4.0-h-small.tomlnew file mode 100644index 00000000..17a1a281--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-granite-4.0-h-small.toml@@ -0,0 +1,30 @@+# liveresearch: ibm-granite/granite-4.0-h-small+[meta]+name = "liveresearch-granite-4.0-h-small"+description = "liveresearch on ibm-granite/granite-4.0-h-small"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 16+output_dir = "results/neurips-2026/baselines/granite-4.0-h-small/liveresearch/"+seed = 42++[[models]]+name = "ibm-granite/granite-4.0-h-small"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+max_samples = 50+agent = "monitor_operative"+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-granite-4.0-micro.toml b/src/openjarvis/evals/configs/liveresearch-granite-4.0-micro.tomlnew file mode 100644index 00000000..d1089b36--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-granite-4.0-micro.toml@@ -0,0 +1,30 @@+# liveresearch: ibm-granite/granite-4.0-micro+[meta]+name = "liveresearch-granite-4.0-micro"+description = "liveresearch on ibm-granite/granite-4.0-micro"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 16+output_dir = "results/neurips-2026/baselines/granite-4.0-micro/liveresearch/"+seed = 42++[[models]]+name = "ibm-granite/granite-4.0-micro"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+max_samples = 50+agent = "monitor_operative"+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml b/src/openjarvis/evals/configs/liveresearch-nemotron-nano.tomlindex 63ecf526..483aa3c2 100644--- a/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml+++ b/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml@@ -1,7 +1,7 @@-# LiveResearchBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)+# DeepResearchBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)[meta]name = "liveresearch-nemotron-nano"-description = "LiveResearchBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 with monitor_operative agent"+description = "DeepResearchBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 with monitor_operative agent"[defaults]temperature = 0.6diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-27b-lora-r32.toml b/src/openjarvis/evals/configs/liveresearch-qwen-27b-lora-r32.tomlnew file mode 100644index 00000000..808d5156--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-qwen-27b-lora-r32.toml@@ -0,0 +1,30 @@+# DeepResearchBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4)+[meta]+name = "liveresearch-qwen-27b-lora-r32"+description = "DeepResearchBench on qwen27b-lora-r32 with monitor_operative agent"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 8+output_dir = "results/neurips-2026/baselines/liveresearch-qwen-27b/"+seed = 42++[[models]]+name = "qwen27b-lora-r32"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-27b.tomlindex 05857481..ecf09b6d 100644--- a/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml+++ b/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml@@ -1,7 +1,7 @@-# LiveResearchBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4)+# DeepResearchBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4)[meta]name = "liveresearch-qwen-27b"-description = "LiveResearchBench on Qwen/Qwen3.5-27B-FP8 with monitor_operative agent"+description = "DeepResearchBench on Qwen/Qwen3.5-27B-FP8 with monitor_operative agent"[defaults]temperature = 0.6diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r16.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r16.tomlnew file mode 100644index 00000000..1cdb806e--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r16.toml@@ -0,0 +1,31 @@+# DeepResearchBench: Qwen3.5-2B (vLLM, 1 GPU)+[meta]+name = "liveresearch-qwen-2b-lora-r16"+description = "DeepResearchBench on Qwen/Qwen3.5-2B"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096+engine = "cloud"++[run]+max_workers = 8+output_dir = "results/neurips-2026/baselines/optimized-eval/track-a/qwen-2b-lora-r16/liveresearch/"+seed = 42++[[models]]+name = "qwen2b-lora-r16"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r32.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r32.tomlnew file mode 100644index 00000000..da96ba7a--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r32.toml@@ -0,0 +1,31 @@+# DeepResearchBench: Qwen3.5-2B (vLLM, 1 GPU)+[meta]+name = "liveresearch-qwen-2b-lora-r32"+description = "DeepResearchBench on Qwen/Qwen3.5-2B"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096+engine = "cloud"++[run]+max_workers = 8+output_dir = "results/neurips-2026/baselines/optimized-eval/track-a/qwen-2b-lora-r32/liveresearch/"+seed = 42++[[models]]+name = "qwen2b-lora-r32"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b-mixed.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b-mixed.tomlnew file mode 100644index 00000000..e9f26e54--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b-mixed.toml@@ -0,0 +1,31 @@+# DeepResearchBench: Qwen3.5-2B (vLLM, 1 GPU)+[meta]+name = "liveresearch-qwen-2b-mixed"+description = "DeepResearchBench on qwen2b-mixed"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096+engine = "cloud"++[run]+max_workers = 8+output_dir = "results/neurips-2026/baselines/optimized-eval/mixed/qwen-2b-lora/liveresearch/"+seed = 42++[[models]]+name = "qwen2b-mixed"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b-trackb.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b-trackb.tomlnew file mode 100644index 00000000..e1777cc8--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b-trackb.toml@@ -0,0 +1,31 @@+# DeepResearchBench: Qwen3.5-2B (vLLM, 1 GPU)+[meta]+name = "liveresearch-qwen-2b-trackb"+description = "DeepResearchBench on qwen2b-trackb"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096+engine = "cloud"++[run]+max_workers = 8+output_dir = "results/neurips-2026/baselines/optimized-eval/track-b/qwen-2b-lora/liveresearch/"+seed = 42++[[models]]+name = "qwen2b-trackb"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b.tomlindex fa27c3e2..dbcd7b45 100644--- a/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml+++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml@@ -1,7 +1,7 @@-# LiveResearchBench: Qwen3.5-2B (vLLM, 1 GPU)+# DeepResearchBench: Qwen3.5-2B (vLLM, 1 GPU)[meta]name = "liveresearch-qwen-2b"-description = "LiveResearchBench on Qwen/Qwen3.5-2B"+description = "DeepResearchBench on Qwen/Qwen3.5-2B"[defaults]temperature = 0.6diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b-lora-r32.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b-lora-r32.tomlnew file mode 100644index 00000000..45e3cf7f--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b-lora-r32.toml@@ -0,0 +1,31 @@+# DeepResearchBench: Qwen3.5-9B (vLLM, 1 GPU)+[meta]+name = "liveresearch-qwen-9b-lora-r32"+description = "DeepResearchBench on Qwen/Qwen3.5-9B"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096+engine = "cloud"++[run]+max_workers = 8+output_dir = "results/neurips-2026/baselines/optimized-eval/track-a/qwen-9b-lora-r32/liveresearch/"+seed = 42++[[models]]+name = "qwen9b-lora-r32"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b-mixed.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b-mixed.tomlnew file mode 100644index 00000000..7c1af65d--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b-mixed.toml@@ -0,0 +1,31 @@+# DeepResearchBench: Qwen3.5-9B (vLLM, 1 GPU)+[meta]+name = "liveresearch-qwen-9b-mixed"+description = "DeepResearchBench on qwen9b-mixed"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096+engine = "cloud"++[run]+max_workers = 8+output_dir = "results/neurips-2026/baselines/optimized-eval/mixed/qwen-9b-lora/liveresearch/"+seed = 42++[[models]]+name = "qwen9b-mixed"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b-trackb.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b-trackb.tomlnew file mode 100644index 00000000..8a802c5c--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b-trackb.toml@@ -0,0 +1,31 @@+# DeepResearchBench: Qwen3.5-9B (vLLM, 1 GPU)+[meta]+name = "liveresearch-qwen-9b-trackb"+description = "DeepResearchBench on qwen9b-trackb"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096+engine = "cloud"++[run]+max_workers = 8+output_dir = "results/neurips-2026/baselines/optimized-eval/track-b/qwen-9b-lora/liveresearch/"+seed = 42++[[models]]+name = "qwen9b-trackb"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b.tomlindex 002e02ae..8da91a75 100644--- a/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml+++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml@@ -1,7 +1,7 @@-# LiveResearchBench: Qwen3.5-9B (vLLM, 1 GPU)+# DeepResearchBench: Qwen3.5-9B (vLLM, 1 GPU)[meta]name = "liveresearch-qwen-9b"-description = "LiveResearchBench on Qwen/Qwen3.5-9B"+description = "DeepResearchBench on Qwen/Qwen3.5-9B"[defaults]temperature = 0.6diff --git a/src/openjarvis/evals/configs/liveresearch-trinity-large.toml b/src/openjarvis/evals/configs/liveresearch-trinity-large.tomlindex 22a5d066..fd2e2214 100644--- a/src/openjarvis/evals/configs/liveresearch-trinity-large.toml+++ b/src/openjarvis/evals/configs/liveresearch-trinity-large.toml@@ -1,7 +1,7 @@-# LiveResearchBench eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8)+# DeepResearchBench eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8)[meta]name = "liveresearch-trinity-large"-description = "LiveResearchBench on arcee-ai/Trinity-Large-Thinking-FP8-Block with monitor_operative agent"+description = "DeepResearchBench on arcee-ai/Trinity-Large-Thinking-FP8-Block with monitor_operative agent"[defaults]temperature = 0.6@@ -16,7 +16,7 @@ max_tokens = 4096max_workers = 16# Trinity-Large is a "thinking" model — needs more turns to reach an answer.# Without this override the agent hits the default max_turns=10 cap on-# ~78% of LiveResearchBench tasks before producing a final answer.+# ~78% of DeepResearchBench tasks before producing a final answer.max_turns = 50output_dir = "results/neurips-2026/baselines/liveresearch-trinity-large/"seed = 42diff --git a/src/openjarvis/evals/datasets/liveresearch.py b/src/openjarvis/evals/datasets/liveresearch.pyindex 100d1427..9eb058f8 100644--- a/src/openjarvis/evals/datasets/liveresearch.py+++ b/src/openjarvis/evals/datasets/liveresearch.py@@ -1,4 +1,4 @@-"""LiveResearchBench dataset provider — deep research benchmark.+"""DeepResearchBench dataset provider — deep research benchmark.Clones the deep_research_bench repo at runtime and parses query + criteriaJSONL files into EvalRecords for use with AgenticRunner.@@ -50,14 +50,14 @@ def _build_criteria_index(class LiveResearchBenchDataset(DatasetProvider):- """LiveResearchBench — deep research with 100 expert-curated tasks.+ """DeepResearchBench — deep research with 100 expert-curated tasks.Clones Ayanami0730/deep_research_bench from GitHub (or uses a localpath) and parses query + criteria JSONL files into EvalRecords."""dataset_id = "liveresearch"- dataset_name = "LiveResearchBench"+ dataset_name = "DeepResearchBench"def __init__(self, path: Optional[str] = None) -> None:self._local_path = Path(path) if path else None@@ -68,7 +68,7 @@ class LiveResearchBenchDataset(DatasetProvider):issues: List[str] = []if self._local_path is None and shutil.which("git") is None:issues.append(- "git binary not found. Install git to clone LiveResearchBench."+ "git binary not found. Install git to clone DeepResearchBench.")return issues@@ -77,12 +77,12 @@ class LiveResearchBenchDataset(DatasetProvider):if self._local_path is not None:if not self._local_path.exists():raise FileNotFoundError(- f"LiveResearchBench path not found: {self._local_path}"+ f"DeepResearchBench path not found: {self._local_path}")return self._local_pathif not self._repo_dir.exists():- LOGGER.info("Cloning LiveResearchBench from %s ...", LIVERESEARCH_REPO)+ LOGGER.info("Cloning DeepResearchBench from %s ...", LIVERESEARCH_REPO)self._repo_dir.parent.mkdir(parents=True, exist_ok=True)subprocess.run([@@ -96,7 +96,7 @@ class LiveResearchBenchDataset(DatasetProvider):check=True,capture_output=True,)- LOGGER.info("LiveResearchBench cloned to %s", self._repo_dir)+ LOGGER.info("DeepResearchBench cloned to %s", self._repo_dir)return self._repo_dir@@ -179,7 +179,7 @@ class LiveResearchBenchDataset(DatasetProvider):))- LOGGER.info("LiveResearchBench: loaded %d tasks", len(self._records))+ LOGGER.info("DeepResearchBench: loaded %d tasks", len(self._records))def iter_records(self) -> Iterable[EvalRecord]:return iter(self._records)diff --git a/src/openjarvis/evals/scorers/liveresearch.py b/src/openjarvis/evals/scorers/liveresearch.pyindex a744fff0..7895e2f1 100644--- a/src/openjarvis/evals/scorers/liveresearch.py+++ b/src/openjarvis/evals/scorers/liveresearch.py@@ -1,7 +1,7 @@-"""LiveResearchBench scorer — LLM-as-judge for deep research quality.+"""DeepResearchBench scorer — LLM-as-judge for deep research quality.Evaluates research output quality across four dimensions from the-LiveResearchBench rubric: comprehensiveness, insight, instruction_following,+DeepResearchBench rubric: comprehensiveness, insight, instruction_following,and readability. Uses LLM-as-judge with per-task criteria when available,falling back to a generic research quality rubric.@@ -21,7 +21,7 @@ from openjarvis.evals.core.types import EvalRecordLOGGER = logging.getLogger(__name__)-# The four scoring dimensions from LiveResearchBench+# The four scoring dimensions from DeepResearchBenchDIMENSIONS = ["comprehensiveness", "insight", "instruction_following", "readability"]# Default dimension weights when task-specific weights are unavailable@@ -240,7 +240,7 @@ def _normalize_response(parsed: Dict[str, Any]) -> Dict[str, Any]:class LiveResearchBenchScorer(LLMJudgeScorer):- """LLM-as-judge scorer for LiveResearchBench deep research tasks.+ """LLM-as-judge scorer for DeepResearchBench deep research tasks.Evaluates research reports across four dimensions:comprehensiveness, insight, instruction_following, readability.diff --git a/tests/evals/test_benchmark_datasets.py b/tests/evals/test_benchmark_datasets.pyindex be7b0aba..5d9f74f8 100644--- a/tests/evals/test_benchmark_datasets.py+++ b/tests/evals/test_benchmark_datasets.py@@ -136,12 +136,12 @@ class TestDatasetInstantiation:assert ds.dataset_id == "livecodebench"assert ds.dataset_name == "LiveCodeBench"- def test_liveresearch(self) -> None:+ def test_deepresearch(self) -> None:from openjarvis.evals.datasets.liveresearch import LiveResearchBenchDatasetds = LiveResearchBenchDataset()assert ds.dataset_id == "liveresearch"- assert ds.dataset_name == "LiveResearchBench"+ assert ds.dataset_name == "DeepResearchBench"def test_toolcall15(self) -> None:from openjarvis.evals.datasets.toolcall15 import ToolCall15Dataset@@ -260,7 +260,7 @@ class TestScorerInstantiation:s = LiveCodeBenchScorer(_mock_backend(), "test-model")assert s.scorer_id == "livecodebench"- def test_liveresearch_scorer(self) -> None:+ def test_deepresearch_scorer(self) -> None:from openjarvis.evals.scorers.liveresearch import LiveResearchBenchScorers = LiveResearchBenchScorer(_mock_backend(), "test-model")