Skip to content

Personal AI, On Personal Devices

Python

fix: rename DeepResearchBench references in docs, configs, and tests

Jon Saad-Falcondb4b031c452f
diff --git a/docs/experiments/neurips-2026-plan.md b/docs/experiments/neurips-2026-plan.md
index 4539b4bf..ccd614f4 100644
--- a/docs/experiments/neurips-2026-plan.md
+++ b/docs/experiments/neurips-2026-plan.md
@@ -117,7 +117,7 @@ Key comparisons:
| terminalbench | TerminalBench | varies | 20 | Implemented |
| toolcall15 | ToolCall-15 | 15 | 15 (all) | TODO |
| livecodebench | LiveCodeBench | ~100 | 20 | TODO |
-| liveresearch | LiveResearchBench | 100 | 10 | TODO |
+| liveresearch | DeepResearchBench | 100 | 10 | TODO |
## Metrics Captured Per Run
- accuracy (benchmark-specific)
@@ -138,7 +138,7 @@ Key comparisons:
### Phase 1a: Implement missing benchmarks
- [ ] ToolCall-15 integration
- [ ] LiveCodeBench integration
-- [ ] LiveResearchBench integration
+- [ ] DeepResearchBench integration
- [ ] Wire telemetry capture to all eval runs
### Phase 1b: Run cloud baselines (no GPU needed)
@@ -148,7 +148,7 @@ Key comparisons:
TauBench Telecom (75%), GAIA (34.29%)
- [x] Gemini 3.1 Pro — PinchBench (78.26%), TauBench A+R (58.33%),
TauBench Telecom (77.5%), GAIA (47.06%)
-- [ ] All 3 cloud baselines — ToolCall-15, LiveCodeBench, LiveResearchBench
+- [ ] All 3 cloud baselines — ToolCall-15, LiveCodeBench, DeepResearchBench
- [ ] All 3 cloud baselines — TerminalBench
### Phase 1c: Run local models (GPU required)
@@ -230,7 +230,7 @@ Training targets:
- Qwen 35B: TauBench telecom + GAIA running
- ToolCall-15 integration: TODO
- LiveCodeBench integration: TODO
-- LiveResearchBench integration: TODO
+- DeepResearchBench integration: TODO
- Telemetry wiring: TODO
### Blocked
diff --git a/src/openjarvis/evals/configs/liveresearch-claude-opus.toml b/src/openjarvis/evals/configs/liveresearch-claude-opus.toml
index 71d173e0..e5bc67d2 100644
--- a/src/openjarvis/evals/configs/liveresearch-claude-opus.toml
+++ b/src/openjarvis/evals/configs/liveresearch-claude-opus.toml
@@ -1,7 +1,7 @@
-# LiveResearchBench: claude-opus-4-6
+# DeepResearchBench: claude-opus-4-6
[meta]
name = "liveresearch-claude-opus"
-description = "LiveResearchBench deep research benchmark on claude-opus-4-6"
+description = "DeepResearchBench deep research benchmark on claude-opus-4-6"
[defaults]
temperature = 0.6
diff --git a/src/openjarvis/evals/configs/liveresearch-gemini-31-pro.toml b/src/openjarvis/evals/configs/liveresearch-gemini-31-pro.toml
new file mode 100644
index 00000000..533bffbe
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-gemini-31-pro.toml
@@ -0,0 +1,29 @@
+# liveresearch: gemini-3.1-pro-preview (cloud)
+[meta]
+name = "liveresearch-gemini-31-pro"
+description = "liveresearch on gemini-3.1-pro-preview"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/gemini-31-pro/liveresearch/"
+seed = 42
+
+[[models]]
+name = "gemini-3.1-pro-preview"
+engine = "cloud"
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+max_samples = 50
+agent = "monitor_operative"
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml b/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml
index a398c7d0..feaba829 100644
--- a/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml
+++ b/src/openjarvis/evals/configs/liveresearch-gemma4-26b.toml
@@ -1,7 +1,7 @@
-# LiveResearchBench: Gemma-4-26B-A4B-it (vLLM, 1 GPU)
+# DeepResearchBench: Gemma-4-26B-A4B-it (vLLM, 1 GPU)
[meta]
name = "liveresearch-gemma4-26b"
-description = "LiveResearchBench on google/gemma-4-26B-A4B-it"
+description = "DeepResearchBench on google/gemma-4-26B-A4B-it"
[defaults]
temperature = 0.6
diff --git a/src/openjarvis/evals/configs/liveresearch-gemma4-31b.toml b/src/openjarvis/evals/configs/liveresearch-gemma4-31b.toml
new file mode 100644
index 00000000..da337db1
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-gemma4-31b.toml
@@ -0,0 +1,30 @@
+# liveresearch: google/gemma-4-31B-it
+[meta]
+name = "liveresearch-gemma4-31b"
+description = "liveresearch on google/gemma-4-31B-it"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 16
+output_dir = "results/neurips-2026/baselines/gemma4-31b/liveresearch/"
+seed = 42
+
+[[models]]
+name = "google/gemma-4-31B-it"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+max_samples = 50
+agent = "monitor_operative"
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-gpt54.toml b/src/openjarvis/evals/configs/liveresearch-gpt54.toml
new file mode 100644
index 00000000..904f32e7
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-gpt54.toml
@@ -0,0 +1,29 @@
+# liveresearch: gpt-5.4-2026-03-05 (cloud)
+[meta]
+name = "liveresearch-gpt54"
+description = "liveresearch on gpt-5.4-2026-03-05"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/gpt54/liveresearch/"
+seed = 42
+
+[[models]]
+name = "gpt-5.4-2026-03-05"
+engine = "cloud"
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+max_samples = 50
+agent = "monitor_operative"
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b-lora.toml b/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b-lora.toml
new file mode 100644
index 00000000..ad3816ca
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b-lora.toml
@@ -0,0 +1,30 @@
+# liveresearch: granite33-agent
+[meta]
+name = "liveresearch-granite-3.3-8b-lora"
+description = "liveresearch on granite33-agent"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 16
+output_dir = "results/neurips-2026/optimized-eval/track-a/granite-3.3-8b-lora-lora/liveresearch/"
+seed = 42
+
+[[models]]
+name = "granite33-agent"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+max_samples = 50
+agent = "monitor_operative"
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b.toml b/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b.toml
new file mode 100644
index 00000000..7a29c69c
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-granite-3.3-8b.toml
@@ -0,0 +1,30 @@
+# liveresearch: ibm-granite/granite-3.3-8b-instruct
+[meta]
+name = "liveresearch-granite-3.3-8b"
+description = "liveresearch on ibm-granite/granite-3.3-8b-instruct"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 16
+output_dir = "results/neurips-2026/baselines/granite-3.3-8b/liveresearch/"
+seed = 42
+
+[[models]]
+name = "ibm-granite/granite-3.3-8b-instruct"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+max_samples = 50
+agent = "monitor_operative"
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-granite-4.0-h-small.toml b/src/openjarvis/evals/configs/liveresearch-granite-4.0-h-small.toml
new file mode 100644
index 00000000..17a1a281
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-granite-4.0-h-small.toml
@@ -0,0 +1,30 @@
+# liveresearch: ibm-granite/granite-4.0-h-small
+[meta]
+name = "liveresearch-granite-4.0-h-small"
+description = "liveresearch on ibm-granite/granite-4.0-h-small"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 16
+output_dir = "results/neurips-2026/baselines/granite-4.0-h-small/liveresearch/"
+seed = 42
+
+[[models]]
+name = "ibm-granite/granite-4.0-h-small"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+max_samples = 50
+agent = "monitor_operative"
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-granite-4.0-micro.toml b/src/openjarvis/evals/configs/liveresearch-granite-4.0-micro.toml
new file mode 100644
index 00000000..d1089b36
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-granite-4.0-micro.toml
@@ -0,0 +1,30 @@
+# liveresearch: ibm-granite/granite-4.0-micro
+[meta]
+name = "liveresearch-granite-4.0-micro"
+description = "liveresearch on ibm-granite/granite-4.0-micro"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 16
+output_dir = "results/neurips-2026/baselines/granite-4.0-micro/liveresearch/"
+seed = 42
+
+[[models]]
+name = "ibm-granite/granite-4.0-micro"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+max_samples = 50
+agent = "monitor_operative"
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml b/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml
index 63ecf526..483aa3c2 100644
--- a/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml
+++ b/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml
@@ -1,7 +1,7 @@
-# LiveResearchBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)
+# DeepResearchBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)
[meta]
name = "liveresearch-nemotron-nano"
-description = "LiveResearchBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 with monitor_operative agent"
+description = "DeepResearchBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 with monitor_operative agent"
[defaults]
temperature = 0.6
diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-27b-lora-r32.toml b/src/openjarvis/evals/configs/liveresearch-qwen-27b-lora-r32.toml
new file mode 100644
index 00000000..808d5156
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-qwen-27b-lora-r32.toml
@@ -0,0 +1,30 @@
+# DeepResearchBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4)
+[meta]
+name = "liveresearch-qwen-27b-lora-r32"
+description = "DeepResearchBench on qwen27b-lora-r32 with monitor_operative agent"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/liveresearch-qwen-27b/"
+seed = 42
+
+[[models]]
+name = "qwen27b-lora-r32"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml
index 05857481..ecf09b6d 100644
--- a/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml
+++ b/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml
@@ -1,7 +1,7 @@
-# LiveResearchBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4)
+# DeepResearchBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4)
[meta]
name = "liveresearch-qwen-27b"
-description = "LiveResearchBench on Qwen/Qwen3.5-27B-FP8 with monitor_operative agent"
+description = "DeepResearchBench on Qwen/Qwen3.5-27B-FP8 with monitor_operative agent"
[defaults]
temperature = 0.6
diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r16.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r16.toml
new file mode 100644
index 00000000..1cdb806e
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r16.toml
@@ -0,0 +1,31 @@
+# DeepResearchBench: Qwen3.5-2B (vLLM, 1 GPU)
+[meta]
+name = "liveresearch-qwen-2b-lora-r16"
+description = "DeepResearchBench on Qwen/Qwen3.5-2B"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+engine = "cloud"
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/optimized-eval/track-a/qwen-2b-lora-r16/liveresearch/"
+seed = 42
+
+[[models]]
+name = "qwen2b-lora-r16"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r32.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r32.toml
new file mode 100644
index 00000000..da96ba7a
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b-lora-r32.toml
@@ -0,0 +1,31 @@
+# DeepResearchBench: Qwen3.5-2B (vLLM, 1 GPU)
+[meta]
+name = "liveresearch-qwen-2b-lora-r32"
+description = "DeepResearchBench on Qwen/Qwen3.5-2B"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+engine = "cloud"
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/optimized-eval/track-a/qwen-2b-lora-r32/liveresearch/"
+seed = 42
+
+[[models]]
+name = "qwen2b-lora-r32"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b-mixed.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b-mixed.toml
new file mode 100644
index 00000000..e9f26e54
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b-mixed.toml
@@ -0,0 +1,31 @@
+# DeepResearchBench: Qwen3.5-2B (vLLM, 1 GPU)
+[meta]
+name = "liveresearch-qwen-2b-mixed"
+description = "DeepResearchBench on qwen2b-mixed"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+engine = "cloud"
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/optimized-eval/mixed/qwen-2b-lora/liveresearch/"
+seed = 42
+
+[[models]]
+name = "qwen2b-mixed"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b-trackb.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b-trackb.toml
new file mode 100644
index 00000000..e1777cc8
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b-trackb.toml
@@ -0,0 +1,31 @@
+# DeepResearchBench: Qwen3.5-2B (vLLM, 1 GPU)
+[meta]
+name = "liveresearch-qwen-2b-trackb"
+description = "DeepResearchBench on qwen2b-trackb"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+engine = "cloud"
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/optimized-eval/track-b/qwen-2b-lora/liveresearch/"
+seed = 42
+
+[[models]]
+name = "qwen2b-trackb"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml
index fa27c3e2..dbcd7b45 100644
--- a/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml
+++ b/src/openjarvis/evals/configs/liveresearch-qwen-2b.toml
@@ -1,7 +1,7 @@
-# LiveResearchBench: Qwen3.5-2B (vLLM, 1 GPU)
+# DeepResearchBench: Qwen3.5-2B (vLLM, 1 GPU)
[meta]
name = "liveresearch-qwen-2b"
-description = "LiveResearchBench on Qwen/Qwen3.5-2B"
+description = "DeepResearchBench on Qwen/Qwen3.5-2B"
[defaults]
temperature = 0.6
diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b-lora-r32.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b-lora-r32.toml
new file mode 100644
index 00000000..45e3cf7f
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b-lora-r32.toml
@@ -0,0 +1,31 @@
+# DeepResearchBench: Qwen3.5-9B (vLLM, 1 GPU)
+[meta]
+name = "liveresearch-qwen-9b-lora-r32"
+description = "DeepResearchBench on Qwen/Qwen3.5-9B"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+engine = "cloud"
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/optimized-eval/track-a/qwen-9b-lora-r32/liveresearch/"
+seed = 42
+
+[[models]]
+name = "qwen9b-lora-r32"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b-mixed.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b-mixed.toml
new file mode 100644
index 00000000..7c1af65d
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b-mixed.toml
@@ -0,0 +1,31 @@
+# DeepResearchBench: Qwen3.5-9B (vLLM, 1 GPU)
+[meta]
+name = "liveresearch-qwen-9b-mixed"
+description = "DeepResearchBench on qwen9b-mixed"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+engine = "cloud"
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/optimized-eval/mixed/qwen-9b-lora/liveresearch/"
+seed = 42
+
+[[models]]
+name = "qwen9b-mixed"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b-trackb.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b-trackb.toml
new file mode 100644
index 00000000..8a802c5c
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b-trackb.toml
@@ -0,0 +1,31 @@
+# DeepResearchBench: Qwen3.5-9B (vLLM, 1 GPU)
+[meta]
+name = "liveresearch-qwen-9b-trackb"
+description = "DeepResearchBench on qwen9b-trackb"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+engine = "cloud"
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/optimized-eval/track-b/qwen-9b-lora/liveresearch/"
+seed = 42
+
+[[models]]
+name = "qwen9b-trackb"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml
index 002e02ae..8da91a75 100644
--- a/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml
+++ b/src/openjarvis/evals/configs/liveresearch-qwen-9b.toml
@@ -1,7 +1,7 @@
-# LiveResearchBench: Qwen3.5-9B (vLLM, 1 GPU)
+# DeepResearchBench: Qwen3.5-9B (vLLM, 1 GPU)
[meta]
name = "liveresearch-qwen-9b"
-description = "LiveResearchBench on Qwen/Qwen3.5-9B"
+description = "DeepResearchBench on Qwen/Qwen3.5-9B"
[defaults]
temperature = 0.6
diff --git a/src/openjarvis/evals/configs/liveresearch-trinity-large.toml b/src/openjarvis/evals/configs/liveresearch-trinity-large.toml
index 22a5d066..fd2e2214 100644
--- a/src/openjarvis/evals/configs/liveresearch-trinity-large.toml
+++ b/src/openjarvis/evals/configs/liveresearch-trinity-large.toml
@@ -1,7 +1,7 @@
-# LiveResearchBench eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8)
+# DeepResearchBench eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8)
[meta]
name = "liveresearch-trinity-large"
-description = "LiveResearchBench on arcee-ai/Trinity-Large-Thinking-FP8-Block with monitor_operative agent"
+description = "DeepResearchBench on arcee-ai/Trinity-Large-Thinking-FP8-Block with monitor_operative agent"
[defaults]
temperature = 0.6
@@ -16,7 +16,7 @@ max_tokens = 4096
max_workers = 16
# Trinity-Large is a "thinking" model — needs more turns to reach an answer.
# Without this override the agent hits the default max_turns=10 cap on
-# ~78% of LiveResearchBench tasks before producing a final answer.
+# ~78% of DeepResearchBench tasks before producing a final answer.
max_turns = 50
output_dir = "results/neurips-2026/baselines/liveresearch-trinity-large/"
seed = 42
diff --git a/src/openjarvis/evals/datasets/liveresearch.py b/src/openjarvis/evals/datasets/liveresearch.py
index 100d1427..9eb058f8 100644
--- a/src/openjarvis/evals/datasets/liveresearch.py
+++ b/src/openjarvis/evals/datasets/liveresearch.py
@@ -1,4 +1,4 @@
-"""LiveResearchBench dataset provider — deep research benchmark.
+"""DeepResearchBench dataset provider — deep research benchmark.
Clones the deep_research_bench repo at runtime and parses query + criteria
JSONL files into EvalRecords for use with AgenticRunner.
@@ -50,14 +50,14 @@ def _build_criteria_index(
class LiveResearchBenchDataset(DatasetProvider):
- """LiveResearchBench — deep research with 100 expert-curated tasks.
+ """DeepResearchBench — deep research with 100 expert-curated tasks.
Clones Ayanami0730/deep_research_bench from GitHub (or uses a local
path) and parses query + criteria JSONL files into EvalRecords.
"""
dataset_id = "liveresearch"
- dataset_name = "LiveResearchBench"
+ dataset_name = "DeepResearchBench"
def __init__(self, path: Optional[str] = None) -> None:
self._local_path = Path(path) if path else None
@@ -68,7 +68,7 @@ class LiveResearchBenchDataset(DatasetProvider):
issues: List[str] = []
if self._local_path is None and shutil.which("git") is None:
issues.append(
- "git binary not found. Install git to clone LiveResearchBench."
+ "git binary not found. Install git to clone DeepResearchBench."
)
return issues
@@ -77,12 +77,12 @@ class LiveResearchBenchDataset(DatasetProvider):
if self._local_path is not None:
if not self._local_path.exists():
raise FileNotFoundError(
- f"LiveResearchBench path not found: {self._local_path}"
+ f"DeepResearchBench path not found: {self._local_path}"
)
return self._local_path
if not self._repo_dir.exists():
- LOGGER.info("Cloning LiveResearchBench from %s ...", LIVERESEARCH_REPO)
+ LOGGER.info("Cloning DeepResearchBench from %s ...", LIVERESEARCH_REPO)
self._repo_dir.parent.mkdir(parents=True, exist_ok=True)
subprocess.run(
[
@@ -96,7 +96,7 @@ class LiveResearchBenchDataset(DatasetProvider):
check=True,
capture_output=True,
)
- LOGGER.info("LiveResearchBench cloned to %s", self._repo_dir)
+ LOGGER.info("DeepResearchBench cloned to %s", self._repo_dir)
return self._repo_dir
@@ -179,7 +179,7 @@ class LiveResearchBenchDataset(DatasetProvider):
)
)
- LOGGER.info("LiveResearchBench: loaded %d tasks", len(self._records))
+ LOGGER.info("DeepResearchBench: loaded %d tasks", len(self._records))
def iter_records(self) -> Iterable[EvalRecord]:
return iter(self._records)
diff --git a/src/openjarvis/evals/scorers/liveresearch.py b/src/openjarvis/evals/scorers/liveresearch.py
index a744fff0..7895e2f1 100644
--- a/src/openjarvis/evals/scorers/liveresearch.py
+++ b/src/openjarvis/evals/scorers/liveresearch.py
@@ -1,7 +1,7 @@
-"""LiveResearchBench scorer — LLM-as-judge for deep research quality.
+"""DeepResearchBench scorer — LLM-as-judge for deep research quality.
Evaluates research output quality across four dimensions from the
-LiveResearchBench rubric: comprehensiveness, insight, instruction_following,
+DeepResearchBench rubric: comprehensiveness, insight, instruction_following,
and readability. Uses LLM-as-judge with per-task criteria when available,
falling back to a generic research quality rubric.
@@ -21,7 +21,7 @@ from openjarvis.evals.core.types import EvalRecord
LOGGER = logging.getLogger(__name__)
-# The four scoring dimensions from LiveResearchBench
+# The four scoring dimensions from DeepResearchBench
DIMENSIONS = ["comprehensiveness", "insight", "instruction_following", "readability"]
# Default dimension weights when task-specific weights are unavailable
@@ -240,7 +240,7 @@ def _normalize_response(parsed: Dict[str, Any]) -> Dict[str, Any]:
class LiveResearchBenchScorer(LLMJudgeScorer):
- """LLM-as-judge scorer for LiveResearchBench deep research tasks.
+ """LLM-as-judge scorer for DeepResearchBench deep research tasks.
Evaluates research reports across four dimensions:
comprehensiveness, insight, instruction_following, readability.
diff --git a/tests/evals/test_benchmark_datasets.py b/tests/evals/test_benchmark_datasets.py
index be7b0aba..5d9f74f8 100644
--- a/tests/evals/test_benchmark_datasets.py
+++ b/tests/evals/test_benchmark_datasets.py
@@ -136,12 +136,12 @@ class TestDatasetInstantiation:
assert ds.dataset_id == "livecodebench"
assert ds.dataset_name == "LiveCodeBench"
- def test_liveresearch(self) -> None:
+ def test_deepresearch(self) -> None:
from openjarvis.evals.datasets.liveresearch import LiveResearchBenchDataset
ds = LiveResearchBenchDataset()
assert ds.dataset_id == "liveresearch"
- assert ds.dataset_name == "LiveResearchBench"
+ assert ds.dataset_name == "DeepResearchBench"
def test_toolcall15(self) -> None:
from openjarvis.evals.datasets.toolcall15 import ToolCall15Dataset
@@ -260,7 +260,7 @@ class TestScorerInstantiation:
s = LiveCodeBenchScorer(_mock_backend(), "test-model")
assert s.scorer_id == "livecodebench"
- def test_liveresearch_scorer(self) -> None:
+ def test_deepresearch_scorer(self) -> None:
from openjarvis.evals.scorers.liveresearch import LiveResearchBenchScorer
s = LiveResearchBenchScorer(_mock_backend(), "test-model")