~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Tests for benchmarks/metrics/calculation.py
Tests cover:
- calculate_metrics function
- calculate_combined_score function
- calculate_resource_metrics function
"""
import json
class TestCalculateMetrics:
"""Tests for the calculate_metrics function."""
def test_calculates_basic_metrics(self, tmp_path):
"""Test calculation of basic metrics from results file."""
from local_deep_research.benchmarks.metrics.calculation import (
calculate_metrics,
)
# Create a test results file
results_file = tmp_path / "results.jsonl"
results = [
{"is_correct": True, "processing_time": 1.5, "confidence": "90"},
{"is_correct": True, "processing_time": 2.0, "confidence": "85"},
{"is_correct": False, "processing_time": 1.0, "confidence": "70"},
]
with open(results_file, "w") as f:
for r in results:
f.write(json.dumps(r) + "\n")
metrics = calculate_metrics(str(results_file))
assert metrics["total_examples"] == 3
assert metrics["graded_examples"] == 3
assert metrics["correct"] == 2
assert metrics["accuracy"] == 2 / 3
assert metrics["average_processing_time"] == 1.5
assert metrics["average_confidence"] == (90 + 85 + 70) / 3
def test_handles_empty_file(self, tmp_path):
"""Test handling of empty results file."""
from local_deep_research.benchmarks.metrics.calculation import (
calculate_metrics,
)
results_file = tmp_path / "empty.jsonl"
results_file.write_text("")
metrics = calculate_metrics(str(results_file))
assert "error" in metrics
assert metrics["error"] == "No results found"
def test_handles_missing_file(self, tmp_path):
"""Test handling of missing results file."""
from local_deep_research.benchmarks.metrics.calculation import (
calculate_metrics,
)
metrics = calculate_metrics(str(tmp_path / "nonexistent.jsonl"))
assert "error" in metrics
def test_calculates_error_rate(self, tmp_path):
"""Test calculation of error rate."""
from local_deep_research.benchmarks.metrics.calculation import (
calculate_metrics,
)
results_file = tmp_path / "results.jsonl"
results = [
{"is_correct": True},
{"error": "Some error occurred"},
{"is_correct": False},
]
with open(results_file, "w") as f:
for r in results:
f.write(json.dumps(r) + "\n")
metrics = calculate_metrics(str(results_file))
assert metrics["error_count"] == 1
assert metrics["error_rate"] == 1 / 3
def test_calculates_per_category_metrics(self, tmp_path):
"""Test calculation of per-category metrics."""
from local_deep_research.benchmarks.metrics.calculation import (
calculate_metrics,
)
results_file = tmp_path / "results.jsonl"
results = [
{"is_correct": True, "category": "science"},
{"is_correct": False, "category": "science"},
{"is_correct": True, "category": "history"},
{"is_correct": True, "category": "history"},
]
with open(results_file, "w") as f:
for r in results:
f.write(json.dumps(r) + "\n")
metrics = calculate_metrics(str(results_file))
assert "categories" in metrics
assert metrics["categories"]["science"]["total"] == 2
assert metrics["categories"]["science"]["correct"] == 1
assert metrics["categories"]["science"]["accuracy"] == 0.5
assert metrics["categories"]["history"]["total"] == 2
assert metrics["categories"]["history"]["correct"] == 2
assert metrics["categories"]["history"]["accuracy"] == 1.0
def test_handles_missing_confidence(self, tmp_path):
"""Test handling of results without confidence values."""
from local_deep_research.benchmarks.metrics.calculation import (
calculate_metrics,
)
results_file = tmp_path / "results.jsonl"
results = [
{"is_correct": True},
{"is_correct": False},
]
with open(results_file, "w") as f:
for r in results:
f.write(json.dumps(r) + "\n")
metrics = calculate_metrics(str(results_file))
assert metrics["average_confidence"] == 0
def test_handles_invalid_confidence(self, tmp_path):
"""Test handling of invalid confidence values."""
from local_deep_research.benchmarks.metrics.calculation import (
calculate_metrics,
)
results_file = tmp_path / "results.jsonl"
results = [
{"is_correct": True, "confidence": "invalid"},
{"is_correct": True, "confidence": "80"},
]
with open(results_file, "w") as f:
for r in results:
f.write(json.dumps(r) + "\n")
metrics = calculate_metrics(str(results_file))
# Should skip invalid and only use valid confidence
assert metrics["average_confidence"] == 80
class TestCalculateCombinedScore:
"""Tests for the calculate_combined_score function."""
def test_calculates_with_default_weights(self):
"""Test combined score with default weights."""
from local_deep_research.benchmarks.metrics.calculation import (
calculate_combined_score,
)
metrics = {
"quality": {"quality_score": 0.8},
"speed": {"speed_score": 0.6},
"resource": {"resource_score": 0.9},
}
score = calculate_combined_score(metrics)
# Default weights: quality=0.6, speed=0.3, resource=0.1
expected = 0.8 * 0.6 + 0.6 * 0.3 + 0.9 * 0.1
assert abs(score - expected) < 0.001
def test_calculates_with_custom_weights(self):
"""Test combined score with custom weights."""
from local_deep_research.benchmarks.metrics.calculation import (
calculate_combined_score,
)
metrics = {
"quality": {"quality_score": 1.0},
"speed": {"speed_score": 0.5},
"resource": {"resource_score": 0.0},
}
weights = {"quality": 0.5, "speed": 0.5, "resource": 0.0}
score = calculate_combined_score(metrics, weights)
expected = 1.0 * 0.5 + 0.5 * 0.5 + 0.0 * 0.0
assert abs(score - expected) < 0.001
def test_handles_missing_metrics(self):
"""Test handling of missing metric categories."""
from local_deep_research.benchmarks.metrics.calculation import (
calculate_combined_score,
)
metrics = {
"quality": {"quality_score": 0.8},
# Missing speed and resource
}
score = calculate_combined_score(metrics)
# Should only count quality component
assert score > 0
assert score <= 1.0
def test_handles_zero_weights(self):
"""Test handling of zero total weight."""
from local_deep_research.benchmarks.metrics.calculation import (
calculate_combined_score,
)
metrics = {
"quality": {"quality_score": 0.8},
}
weights = {"quality": 0, "speed": 0, "resource": 0}
score = calculate_combined_score(metrics, weights)
assert score == 0.0
def test_normalizes_weights(self):
"""Test that weights are normalized."""
from local_deep_research.benchmarks.metrics.calculation import (
calculate_combined_score,
)
metrics = {
"quality": {"quality_score": 1.0},
"speed": {"speed_score": 1.0},
"resource": {"resource_score": 1.0},
}
# Weights that don't sum to 1
weights = {"quality": 2, "speed": 1, "resource": 1}
score = calculate_combined_score(metrics, weights)
# All scores are 1.0, so normalized result should be 1.0
assert abs(score - 1.0) < 0.001
class TestCalculateResourceMetrics:
"""Tests for the calculate_resource_metrics function."""
def test_calculates_resource_score(self):
"""Test calculation of resource score."""
from local_deep_research.benchmarks.metrics.calculation import (
calculate_resource_metrics,
)
config = {
"iterations": 2,
"questions_per_iteration": 2,
"max_results": 50,
}
metrics = calculate_resource_metrics(config)
assert "resource_score" in metrics
assert "estimated_complexity" in metrics
assert 0 <= metrics["resource_score"] <= 1
def test_higher_complexity_lower_score(self):
"""Test that higher complexity results in lower resource score."""
from local_deep_research.benchmarks.metrics.calculation import (
calculate_resource_metrics,
)
low_complexity_config = {
"iterations": 1,
"questions_per_iteration": 1,
"max_results": 25,
}
high_complexity_config = {
"iterations": 5,
"questions_per_iteration": 5,
"max_results": 100,
}
low_metrics = calculate_resource_metrics(low_complexity_config)
high_metrics = calculate_resource_metrics(high_complexity_config)
assert low_metrics["resource_score"] > high_metrics["resource_score"]
assert (
low_metrics["estimated_complexity"]
< high_metrics["estimated_complexity"]
)
def test_uses_default_values(self):
"""Test that default values are used for missing config."""
from local_deep_research.benchmarks.metrics.calculation import (
calculate_resource_metrics,
)
empty_config = {}
metrics = calculate_resource_metrics(empty_config)
assert "resource_score" in metrics
assert metrics["resource_score"] > 0