~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Tests for benchmarks/comparison/evaluator.py covering missing lines.
Covers:
- Lines 539-543: Duration conversion (hours, minutes) in _create_metric_comparison_chart
- Lines 599-652: Spider chart RadarAxes creation (polygon frame, unit_poly_verts)
- Lines 723-736: Spider chart exception fallback
- Lines 783-784: Pareto chart dominated point detection
- _calculate_average_metrics with speed/resource/quality metrics
"""
from pathlib import Path
from unittest.mock import patch
MODULE = "local_deep_research.benchmarks.comparison.evaluator"
# ---------------------------------------------------------------------------
# _calculate_average_metrics
# ---------------------------------------------------------------------------
class TestCalculateAverageMetrics:
def test_averages_quality_metrics(self):
from local_deep_research.benchmarks.comparison.evaluator import (
_calculate_average_metrics,
)
results = [
{
"success": True,
"quality_metrics": {"accuracy": 0.8, "completeness": 0.6},
"speed_metrics": {},
"resource_metrics": {},
},
{
"success": True,
"quality_metrics": {"accuracy": 0.9, "completeness": 0.7},
"speed_metrics": {},
"resource_metrics": {},
},
]
avg = _calculate_average_metrics(results)
assert abs(avg["quality_metrics"]["accuracy"] - 0.85) < 0.01
assert abs(avg["quality_metrics"]["completeness"] - 0.65) < 0.01
def test_averages_speed_metrics(self):
from local_deep_research.benchmarks.comparison.evaluator import (
_calculate_average_metrics,
)
results = [
{
"success": True,
"quality_metrics": {},
"speed_metrics": {"total_duration": 120},
"resource_metrics": {},
},
{
"success": True,
"quality_metrics": {},
"speed_metrics": {"total_duration": 180},
"resource_metrics": {},
},
]
avg = _calculate_average_metrics(results)
assert avg["speed_metrics"]["total_duration"] == 150.0
def test_averages_resource_metrics(self):
from local_deep_research.benchmarks.comparison.evaluator import (
_calculate_average_metrics,
)
results = [
{
"success": True,
"quality_metrics": {},
"speed_metrics": {},
"resource_metrics": {"token_count": 1000},
},
{
"success": True,
"quality_metrics": {},
"speed_metrics": {},
"resource_metrics": {"token_count": 2000},
},
]
avg = _calculate_average_metrics(results)
assert avg["resource_metrics"]["token_count"] == 1500.0
def test_skips_none_values(self):
from local_deep_research.benchmarks.comparison.evaluator import (
_calculate_average_metrics,
)
results = [
{
"success": True,
"quality_metrics": {"accuracy": 0.9},
"speed_metrics": {},
"resource_metrics": {},
},
{
"success": True,
"quality_metrics": {},
"speed_metrics": {},
"resource_metrics": {},
},
]
avg = _calculate_average_metrics(results)
assert avg["quality_metrics"]["accuracy"] == 0.9
# ---------------------------------------------------------------------------
# _create_metric_comparison_chart: duration conversion (lines 538-543)
# ---------------------------------------------------------------------------
class TestMetricComparisonDurationConversion:
def test_duration_hours_conversion(self, tmp_path):
"""Duration > 3600 is converted to hours."""
from local_deep_research.benchmarks.comparison.evaluator import (
_create_metric_comparison_chart,
)
results = [
{
"config_name": "Config A",
"avg_metrics": {
"speed_metrics": {"total_duration": 7200},
},
},
{
"config_name": "Config B",
"avg_metrics": {
"speed_metrics": {"total_duration": 3601},
},
},
]
output = str(tmp_path / "chart.png")
_create_metric_comparison_chart(
results,
["Config A", "Config B"],
["total_duration"],
"speed_metrics",
"Duration Test",
output,
)
assert Path(output).exists()
def test_duration_minutes_conversion(self, tmp_path):
"""Duration > 60 but < 3600 is converted to minutes."""
from local_deep_research.benchmarks.comparison.evaluator import (
_create_metric_comparison_chart,
)
results = [
{
"config_name": "Config A",
"avg_metrics": {
"speed_metrics": {"total_duration": 120},
},
},
]
output = str(tmp_path / "chart.png")
_create_metric_comparison_chart(
results,
["Config A"],
["total_duration"],
"speed_metrics",
"Duration Test",
output,
)
assert Path(output).exists()
def test_duration_seconds_no_conversion(self, tmp_path):
"""Duration <= 60 stays in seconds."""
from local_deep_research.benchmarks.comparison.evaluator import (
_create_metric_comparison_chart,
)
results = [
{
"config_name": "Config A",
"avg_metrics": {
"speed_metrics": {"total_duration": 30},
},
},
]
output = str(tmp_path / "chart.png")
_create_metric_comparison_chart(
results,
["Config A"],
["total_duration"],
"speed_metrics",
"Duration Test",
output,
)
assert Path(output).exists()
# ---------------------------------------------------------------------------
# _create_spider_chart: exception fallback (lines 723-736)
# ---------------------------------------------------------------------------
class TestSpiderChartExceptionFallback:
def test_spider_chart_exception_creates_fallback_text(self, tmp_path):
"""When spider chart creation fails, a text-based fallback is saved."""
from local_deep_research.benchmarks.comparison.evaluator import (
_create_spider_chart,
)
results = [
{
"config_name": "A",
"avg_metrics": {"quality_metrics": {}},
}
]
output = str(tmp_path / "spider.png")
with patch(
f"{MODULE}.np.linspace",
side_effect=RuntimeError("radar fail"),
):
_create_spider_chart(results, ["A"], output)
assert Path(output).exists()
# ---------------------------------------------------------------------------
# _create_pareto_chart: dominated point detection (lines 783-784)
# ---------------------------------------------------------------------------
class TestParetoChartDominatedPoints:
def test_pareto_with_dominated_points(self, tmp_path):
"""Points dominated by others are not on the Pareto frontier."""
from local_deep_research.benchmarks.comparison.evaluator import (
_create_pareto_chart,
)
results = [
{
"config_name": "Fast Low Quality",
"avg_metrics": {
"quality_metrics": {"overall_quality": 0.3},
"speed_metrics": {"total_duration": 10},
},
},
{
"config_name": "Best",
"avg_metrics": {
"quality_metrics": {"overall_quality": 0.9},
"speed_metrics": {"total_duration": 5},
},
},
{
"config_name": "Dominated",
"avg_metrics": {
"quality_metrics": {"overall_quality": 0.5},
"speed_metrics": {"total_duration": 20},
},
},
]
output = str(tmp_path / "pareto.png")
_create_pareto_chart(results, output)
assert Path(output).exists()
def test_pareto_all_equal(self, tmp_path):
"""When all points are equal, all are on the frontier."""
from local_deep_research.benchmarks.comparison.evaluator import (
_create_pareto_chart,
)
results = [
{
"config_name": f"Config {i}",
"avg_metrics": {
"quality_metrics": {"overall_quality": 0.5},
"speed_metrics": {"total_duration": 10},
},
}
for i in range(3)
]
output = str(tmp_path / "pareto_equal.png")
_create_pareto_chart(results, output)
assert Path(output).exists()