Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

feat(benchmarks): add statistical functions module (#4029)

Ishitta2808f0fa9dd7
diff --git a/src/local_deep_research/benchmarks/metrics/__init__.py b/src/local_deep_research/benchmarks/metrics/__init__.py
index 50001183cf..db2b3efbfe 100644
--- a/src/local_deep_research/benchmarks/metrics/__init__.py
+++ b/src/local_deep_research/benchmarks/metrics/__init__.py
@@ -13,6 +13,10 @@ from .calculation import (
calculate_speed_metrics,
)
from .reporting import generate_report
+from .statistics import (
+ sample_size_for_difference,
+ wilson_score_interval,
+)
__all__ = [
"calculate_combined_score",
@@ -21,4 +25,6 @@ __all__ = [
"calculate_resource_metrics",
"calculate_speed_metrics",
"generate_report",
+ "sample_size_for_difference",
+ "wilson_score_interval",
]
diff --git a/src/local_deep_research/benchmarks/metrics/calculation.py b/src/local_deep_research/benchmarks/metrics/calculation.py
index c2558c2724..f288acb3ef 100644
--- a/src/local_deep_research/benchmarks/metrics/calculation.py
+++ b/src/local_deep_research/benchmarks/metrics/calculation.py
@@ -38,12 +38,17 @@ def calculate_metrics(results_file: str) -> Dict[str, Any]:
if not results:
return {"error": "No results found"}
+ from .statistics import wilson_score_interval
+
# Calculate accuracy
graded_results = [r for r in results if "is_correct" in r]
correct_count = sum(1 for r in graded_results if r.get("is_correct", False))
total_graded = len(graded_results)
accuracy = correct_count / total_graded if total_graded else 0
+ # Wilson score confidence interval for accuracy
+ accuracy_ci = wilson_score_interval(correct_count, total_graded)
+
# Calculate average processing time if available
processing_times = [
r.get("processing_time", 0) for r in results if "processing_time" in r
@@ -77,6 +82,7 @@ def calculate_metrics(results_file: str) -> Dict[str, Any]:
"graded_examples": total_graded,
"correct": correct_count,
"accuracy": accuracy,
+ "accuracy_ci": accuracy_ci,
"average_processing_time": avg_time,
"average_confidence": avg_confidence,
"error_count": error_count,
@@ -98,13 +104,15 @@ def calculate_metrics(results_file: str) -> Dict[str, Any]:
if categories:
category_metrics = {}
for category, counts in categories.items():
+ cat_accuracy = (
+ counts["correct"] / counts["total"] if counts["total"] else 0
+ )
category_metrics[category] = {
"total": counts["total"],
"correct": counts["correct"],
- "accuracy": (
- counts["correct"] / counts["total"]
- if counts["total"]
- else 0
+ "accuracy": cat_accuracy,
+ "accuracy_ci": wilson_score_interval(
+ counts["correct"], counts["total"]
),
}
metrics["categories"] = category_metrics
diff --git a/src/local_deep_research/benchmarks/metrics/reporting.py b/src/local_deep_research/benchmarks/metrics/reporting.py
index 39051d091e..e40e842a00 100644
--- a/src/local_deep_research/benchmarks/metrics/reporting.py
+++ b/src/local_deep_research/benchmarks/metrics/reporting.py
@@ -65,6 +65,14 @@ def generate_report(
f"- **Accuracy**: {metrics.get('accuracy', 0):.3f}",
]
+ # Add confidence interval if available
+ accuracy_ci = metrics.get("accuracy_ci")
+ if accuracy_ci and accuracy_ci.get("sample_size", 0) > 0:
+ report.append(
+ f"- **95% CI**: [{accuracy_ci['lower']:.3f}, {accuracy_ci['upper']:.3f}] "
+ f"(Wilson score, n={accuracy_ci['sample_size']})"
+ )
+
if "average_processing_time" in metrics:
report.append(
f"- **Average Processing Time**: {metrics['average_processing_time']:.2f} seconds"
@@ -91,6 +99,11 @@ def generate_report(
report.append(f"- **Total**: {category_metrics['total']}")
report.append(f"- **Correct**: {category_metrics['correct']}")
report.append(f"- **Accuracy**: {category_metrics['accuracy']:.3f}")
+ cat_ci = category_metrics.get("accuracy_ci")
+ if cat_ci and cat_ci.get("sample_size", 0) > 0:
+ report.append(
+ f"- **95% CI**: [{cat_ci['lower']:.3f}, {cat_ci['upper']:.3f}]"
+ )
report.append("")
# Add configuration info if provided
diff --git a/src/local_deep_research/benchmarks/metrics/statistics.py b/src/local_deep_research/benchmarks/metrics/statistics.py
new file mode 100644
index 0000000000..d58ce3bfdd
--- /dev/null
+++ b/src/local_deep_research/benchmarks/metrics/statistics.py
@@ -0,0 +1,211 @@
+"""
+Statistical functions for benchmark evaluation.
+
+Provides confidence intervals, sample size calculations, and
+proportion statistics for interpreting benchmark results.
+All implementations are pure Python.
+"""
+
+import math
+
+
+def normal_quantile(p: float) -> float:
+ """
+ Inverse normal CDF (percent-point function).
+
+ Uses the Beasley-Springer-Moro rational approximation,
+ accurate to approximately 1e-8.
+
+ Args:
+ p: Probability in (0, 1).
+
+ Returns:
+ z such that P(Z <= z) = p for standard normal Z.
+
+ Raises:
+ ValueError: If p is not in (0, 1).
+ """
+ if p <= 0 or p >= 1:
+ raise ValueError(f"p must be in (0, 1), got {p}")
+
+ # Rational approximation coefficients (Beasley-Springer-Moro)
+ a = [
+ -3.969683028665376e01,
+ 2.209460984245205e02,
+ -2.759285104469687e02,
+ 1.383577518672690e02,
+ -3.066479806614716e01,
+ 2.506628277459239e00,
+ ]
+ b = [
+ -5.447609879822406e01,
+ 1.615858368580409e02,
+ -1.556989798598866e02,
+ 6.680131188771972e01,
+ -1.328068155288572e01,
+ ]
+ c = [
+ -7.784894002430293e-03,
+ -3.223964580411365e-01,
+ -2.400758277161838e00,
+ -2.549732539343734e00,
+ 4.374664141464968e00,
+ 2.938163982698783e00,
+ ]
+ d = [
+ 7.784695709041462e-03,
+ 3.224671290700398e-01,
+ 2.445134137142996e00,
+ 3.754408661907416e00,
+ ]
+
+ p_low = 0.02425
+ p_high = 1 - p_low
+
+ if p < p_low:
+ # Lower tail
+ q = math.sqrt(-2 * math.log(p))
+ return (
+ ((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]
+ ) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1)
+ if p <= p_high:
+ # Central region
+ q = p - 0.5
+ r = q * q
+ return (
+ (
+ ((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r
+ + a[5]
+ )
+ * q
+ ) / (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1)
+ # Upper tail
+ q = math.sqrt(-2 * math.log(1 - p))
+ return -(
+ ((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]
+ ) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1)
+
+
+def wilson_score_interval(
+ successes: int,
+ total: int,
+ confidence: float = 0.95,
+) -> dict:
+ """
+ Wilson score confidence interval for a binomial proportion.
+
+ Unlike the normal (Wald) approximation, the Wilson interval:
+ - Never produces bounds outside [0, 1]
+ - Behaves correctly at 0% and 100% accuracy
+ - Has better coverage at small sample sizes
+
+ Args:
+ successes: Number of successes (correct answers).
+ total: Total number of trials (graded examples).
+ confidence: Confidence level (default 0.95 for 95% CI).
+
+ Returns:
+ Dictionary with keys:
+ - lower: Lower bound of the interval
+ - upper: Upper bound of the interval
+ - center: Center of the Wilson interval (not the raw proportion)
+ - margin_of_error: Half-width of the interval
+ - sample_size: The total n used
+ """
+ if total <= 0:
+ return {
+ "lower": 0.0,
+ "upper": 0.0,
+ "center": 0.0,
+ "margin_of_error": 0.0,
+ "sample_size": 0,
+ }
+
+ if successes < 0 or successes > total:
+ raise ValueError(
+ f"successes must be in [0, total], got successes={successes}, total={total}"
+ )
+
+ p_hat = successes / total
+ z = normal_quantile(1 - (1 - confidence) / 2)
+ z2 = z * z
+
+ denominator = 1 + z2 / total
+ center = (p_hat + z2 / (2 * total)) / denominator
+ half_width = (
+ z * math.sqrt(p_hat * (1 - p_hat) / total + z2 / (4 * total * total))
+ ) / denominator
+
+ lower = max(0.0, center - half_width)
+ upper = min(1.0, center + half_width)
+
+ return {
+ "lower": lower,
+ "upper": upper,
+ "center": center,
+ "margin_of_error": half_width,
+ "sample_size": total,
+ }
+
+
+def proportion_std_error(p: float, n: int) -> float:
+ """
+ Standard error of a sample proportion.
+
+ Args:
+ p: Observed proportion (0 to 1).
+ n: Sample size.
+
+ Returns:
+ Standard error, or 0.0 if n <= 0.
+ """
+ if n <= 0:
+ return 0.0
+ if p < 0 or p > 1:
+ raise ValueError(f"p must be in [0, 1], got {p}")
+ return math.sqrt(p * (1 - p) / n)
+
+
+def sample_size_for_difference(
+ p1: float,
+ p2: float,
+ power: float = 0.8,
+ alpha: float = 0.05,
+) -> int:
+ """
+ Required sample size per group to detect a difference between
+ two independent proportions.
+
+ Uses the formula for a two-sided two-proportion z-test:
+ n = (z_alpha/2 + z_beta)^2 * (p1(1-p1) + p2(1-p2)) / (p1 - p2)^2
+
+ Args:
+ p1: Expected proportion for group 1.
+ p2: Expected proportion for group 2.
+ power: Statistical power (default 0.8 = 80%).
+ alpha: Significance level (default 0.05 for two-sided test).
+
+ Returns:
+ Required sample size per group (rounded up).
+
+ Raises:
+ ValueError: If p1 == p2 (no difference to detect).
+ """
+ if p1 == p2:
+ raise ValueError(
+ "p1 and p2 must differ to compute required sample size"
+ )
+ for name, val in [("p1", p1), ("p2", p2)]:
+ if val < 0 or val > 1:
+ raise ValueError(f"{name} must be in [0, 1], got {val}")
+ for name, val in [("power", power), ("alpha", alpha)]:
+ if val <= 0 or val >= 1:
+ raise ValueError(f"{name} must be in (0, 1), got {val}")
+
+ z_alpha = normal_quantile(1 - alpha / 2)
+ z_beta = normal_quantile(power)
+
+ numerator = (z_alpha + z_beta) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2))
+ denominator = (p1 - p2) ** 2
+
+ return math.ceil(numerator / denominator)
diff --git a/src/local_deep_research/benchmarks/web_api/benchmark_service.py b/src/local_deep_research/benchmarks/web_api/benchmark_service.py
index e0f56c09e1..f70db309db 100644
--- a/src/local_deep_research/benchmarks/web_api/benchmark_service.py
+++ b/src/local_deep_research/benchmarks/web_api/benchmark_service.py
@@ -1362,21 +1362,19 @@ class BenchmarkService:
f"estimated_remaining: {estimated_time_remaining:.2f}s"
)
- # Calculate accuracy confidence interval (95% confidence)
+ # Calculate accuracy confidence interval (Wilson score, 95%)
if results and len(results) >= 3:
- import math
+ from local_deep_research.benchmarks.metrics.statistics import (
+ wilson_score_interval,
+ )
n = len(results)
- p = running_accuracy / 100 if running_accuracy else 0
- # Standard error for proportion
- se = math.sqrt(p * (1 - p) / n)
- # 95% confidence interval (±1.96 * SE)
- margin_of_error = 1.96 * se * 100
- _running_acc = running_accuracy or 0.0
+ correct = sum(1 for r in results if r.is_correct)
+ ci = wilson_score_interval(correct, n)
accuracy_confidence = {
- "lower_bound": max(0, _running_acc - margin_of_error),
- "upper_bound": min(100, _running_acc + margin_of_error),
- "margin_of_error": margin_of_error,
+ "lower_bound": ci["lower"] * 100,
+ "upper_bound": ci["upper"] * 100,
+ "margin_of_error": ci["margin_of_error"] * 100,
"sample_size": n,
}
diff --git a/tests/benchmarks/test_statistics.py b/tests/benchmarks/test_statistics.py
new file mode 100644
index 0000000000..d1b56d007f
--- /dev/null
+++ b/tests/benchmarks/test_statistics.py
@@ -0,0 +1,364 @@
+"""
+Tests for benchmarks/metrics/statistics.py
+
+Tests cover:
+- normal_quantile inverse CDF
+- wilson_score_interval confidence intervals
+- proportion_std_error
+- sample_size_for_difference power analysis
+- Integration: calculate_metrics returns accuracy_ci
+"""
+
+import json
+
+import pytest
+
+
+class TestNormalQuantile:
+ """Tests for the normal_quantile function."""
+
+ def test_median(self):
+ from local_deep_research.benchmarks.metrics.statistics import (
+ normal_quantile,
+ )
+
+ assert abs(normal_quantile(0.5) - 0.0) < 1e-6
+
+ def test_975_gives_196(self):
+ from local_deep_research.benchmarks.metrics.statistics import (
+ normal_quantile,
+ )
+
+ assert abs(normal_quantile(0.975) - 1.96) < 0.001
+
+ def test_025_gives_negative_196(self):
+ from local_deep_research.benchmarks.metrics.statistics import (
+ normal_quantile,
+ )
+
+ assert abs(normal_quantile(0.025) - (-1.96)) < 0.001
+
+ def test_symmetry(self):
+ from local_deep_research.benchmarks.metrics.statistics import (
+ normal_quantile,
+ )
+
+ for p in [0.01, 0.05, 0.1, 0.25]:
+ assert abs(normal_quantile(p) + normal_quantile(1 - p)) < 1e-6
+
+ def test_raises_on_zero(self):
+ from local_deep_research.benchmarks.metrics.statistics import (
+ normal_quantile,
+ )
+
+ with pytest.raises(ValueError):
+ normal_quantile(0.0)
+
+ def test_raises_on_one(self):
+ from local_deep_research.benchmarks.metrics.statistics import (
+ normal_quantile,
+ )
+
+ with pytest.raises(ValueError):
+ normal_quantile(1.0)
+
+ def test_raises_on_negative(self):
+ from local_deep_research.benchmarks.metrics.statistics import (
+ normal_quantile,
+ )
+
+ with pytest.raises(ValueError):
+ normal_quantile(-0.5)
+
+ def test_extreme_tails(self):
+ from local_deep_research.benchmarks.metrics.statistics import (
+ normal_quantile,
+ )
+
+ # p=0.001 -> z ~ -3.09
+ assert abs(normal_quantile(0.001) - (-3.09)) < 0.01
+ # p=0.999 -> z ~ 3.09
+ assert abs(normal_quantile(0.999) - 3.09) < 0.01
+
+
+class TestWilsonScoreInterval:
+ """Tests for the wilson_score_interval function."""
+
+ def test_91_of_100(self):
+ """91/100 at 95% should give approximately [0.837, 0.955]."""
+ from local_deep_research.benchmarks.metrics.statistics import (
+ wilson_score_interval,
+ )
+
+ ci = wilson_score_interval(91, 100)
+ assert abs(ci["lower"] - 0.837) < 0.005
+ assert abs(ci["upper"] - 0.955) < 0.005
+ assert ci["sample_size"] == 100
+
+ def test_perfect_score(self):
+ """100/100 should NOT give [1.0, 1.0] (unlike normal approx)."""
+ from local_deep_research.benchmarks.metrics.statistics import (
+ wilson_score_interval,
+ )
+
+ ci = wilson_score_interval(100, 100)
+ assert ci["upper"] == 1.0
+ assert ci["lower"] < 1.0 # Wilson correctly shows uncertainty
+ assert ci["lower"] > 0.95
+
+ def test_zero_score(self):
+ """0/100 should NOT give [0.0, 0.0] (unlike normal approx)."""
+ from local_deep_research.benchmarks.metrics.statistics import (
+ wilson_score_interval,
+ )
+
+ ci = wilson_score_interval(0, 100)
+ assert ci["lower"] == 0.0
+ assert ci["upper"] > 0.0 # Wilson correctly shows uncertainty
+ assert ci["upper"] < 0.05
+
+ def test_zero_total(self):
+ """0/0 should return zeros without error."""
+ from local_deep_research.benchmarks.metrics.statistics import (
+ wilson_score_interval,
+ )
+
+ ci = wilson_score_interval(0, 0)
+ assert ci["lower"] == 0.0
+ assert ci["upper"] == 0.0
+ assert ci["margin_of_error"] == 0.0
+ assert ci["sample_size"] == 0
+
+ def test_raises_on_successes_greater_than_total(self):
+ """successes > total should raise ValueError."""
+ from local_deep_research.benchmarks.metrics.statistics import (
+ wilson_score_interval,
+ )
+
+ with pytest.raises(ValueError, match="successes must be in"):
+ wilson_score_interval(110, 100)
+
+ def test_raises_on_negative_successes(self):
+ """Negative successes should raise ValueError."""
+ from local_deep_research.benchmarks.metrics.statistics import (
+ wilson_score_interval,
+ )
+
+ with pytest.raises(ValueError, match="successes must be in"):
+ wilson_score_interval(-1, 100)
+
+ def test_single_observation(self):
+ """1/1 should give a wide interval."""
+ from local_deep_research.benchmarks.metrics.statistics import (
+ wilson_score_interval,
+ )
+
+ ci = wilson_score_interval(1, 1)
+ assert ci["lower"] < 0.5
+ assert ci["upper"] == 1.0
+
+ def test_bounds_within_0_1(self):
+ """Bounds should always be in [0, 1]."""
+ from local_deep_research.benchmarks.metrics.statistics import (
+ wilson_score_interval,
+ )
+
+ for successes, total in [
+ (0, 10),
+ (10, 10),
+ (5, 10),
+ (1, 100),
+ (99, 100),
+ ]:
+ ci = wilson_score_interval(successes, total)
+ assert 0.0 <= ci["lower"] <= ci["upper"] <= 1.0
+
+ def test_higher_n_narrower_interval(self):
+ """Larger sample size should give narrower CI at same proportion."""
+ from local_deep_research.benchmarks.metrics.statistics import (
+ wilson_score_interval,
+ )
+
+ ci_small = wilson_score_interval(9, 10)
+ ci_large = wilson_score_interval(90, 100)
+ assert ci_large["margin_of_error"] < ci_small["margin_of_error"]
+
+ def test_90_confidence_narrower_than_99(self):
+ """90% CI should be narrower than 99% CI."""
+ from local_deep_research.benchmarks.metrics.statistics import (
+ wilson_score_interval,
+ )
+
+ ci_90 = wilson_score_interval(50, 100, confidence=0.90)
+ ci_99 = wilson_score_interval(50, 100, confidence=0.99)
+ assert ci_90["margin_of_error"] < ci_99["margin_of_error"]
+
+
+class TestProportionStdError:
+ """Tests for the proportion_std_error function."""
+
+ def test_basic(self):
+ from local_deep_research.benchmarks.metrics.statistics import (
+ proportion_std_error,
+ )
+
+ # p=0.5, n=100 -> sqrt(0.25/100) = 0.05
+ assert abs(proportion_std_error(0.5, 100) - 0.05) < 1e-10
+
+ def test_zero_n(self):
+ from local_deep_research.benchmarks.metrics.statistics import (
+ proportion_std_error,
+ )
+
+ assert proportion_std_error(0.5, 0) == 0.0
+
+ def test_extreme_proportions(self):
+ from local_deep_research.benchmarks.metrics.statistics import (
+ proportion_std_error,
+ )
+
+ assert proportion_std_error(0.0, 100) == 0.0
+ assert proportion_std_error(1.0, 100) == 0.0
+
+ def test_raises_on_invalid_p(self):
+ from local_deep_research.benchmarks.metrics.statistics import (
+ proportion_std_error,
+ )
+
+ with pytest.raises(ValueError, match="p must be in"):
+ proportion_std_error(-0.1, 100)
+ with pytest.raises(ValueError, match="p must be in"):
+ proportion_std_error(1.5, 100)
+
+
+class TestSampleSizeForDifference:
+ """Tests for the sample_size_for_difference function."""
+
+ def test_5pp_difference(self):
+ """5pp difference (85% vs 90%) should need ~680 per group."""
+ from local_deep_research.benchmarks.metrics.statistics import (
+ sample_size_for_difference,
+ )
+
+ n = sample_size_for_difference(0.85, 0.90)
+ assert 600 <= n <= 760
+
+ def test_10pp_difference(self):
+ """10pp difference (80% vs 90%) should need ~200 per group."""
+ from local_deep_research.benchmarks.metrics.statistics import (
+ sample_size_for_difference,
+ )
+
+ n = sample_size_for_difference(0.80, 0.90)
+ assert 170 <= n <= 230
+
+ def test_15pp_difference(self):
+ """15pp difference (75% vs 90%) should need ~90 per group."""
+ from local_deep_research.benchmarks.metrics.statistics import (
+ sample_size_for_difference,
+ )
+
+ n = sample_size_for_difference(0.75, 0.90)
+ assert 75 <= n <= 105
+
+ def test_raises_on_equal_proportions(self):
+ from local_deep_research.benchmarks.metrics.statistics import (
+ sample_size_for_difference,
+ )
+
+ with pytest.raises(ValueError):
+ sample_size_for_difference(0.9, 0.9)
+
+ def test_raises_on_invalid_proportions(self):
+ from local_deep_research.benchmarks.metrics.statistics import (
+ sample_size_for_difference,
+ )
+
+ with pytest.raises(ValueError, match="p1 must be in"):
+ sample_size_for_difference(-0.1, 0.9)
+ with pytest.raises(ValueError, match="p2 must be in"):
+ sample_size_for_difference(0.8, 1.5)
+
+ def test_raises_on_invalid_power_or_alpha(self):
+ from local_deep_research.benchmarks.metrics.statistics import (
+ sample_size_for_difference,
+ )
+
+ with pytest.raises(ValueError, match="power must be in"):
+ sample_size_for_difference(0.8, 0.9, power=0.0)
+ with pytest.raises(ValueError, match="alpha must be in"):
+ sample_size_for_difference(0.8, 0.9, alpha=1.0)
+
+ def test_higher_power_needs_more_samples(self):
+ from local_deep_research.benchmarks.metrics.statistics import (
+ sample_size_for_difference,
+ )
+
+ n_80 = sample_size_for_difference(0.80, 0.90, power=0.80)
+ n_95 = sample_size_for_difference(0.80, 0.90, power=0.95)
+ assert n_95 > n_80
+
+
+class TestCalculateMetricsIntegration:
+ """Test that calculate_metrics() returns accuracy_ci (backward compat)."""
+
+ def test_accuracy_ci_present(self, tmp_path):
+ from local_deep_research.benchmarks.metrics.calculation import (
+ calculate_metrics,
+ )
+
+ results_file = tmp_path / "results.jsonl"
+ results = [
+ {"is_correct": True, "processing_time": 1.5},
+ {"is_correct": True, "processing_time": 2.0},
+ {"is_correct": False, "processing_time": 1.0},
+ ]
+ with open(results_file, "w") as f:
+ for r in results:
+ f.write(json.dumps(r) + "\n")
+
+ metrics = calculate_metrics(str(results_file))
+
+ # Existing keys still present
+ assert metrics["accuracy"] == 2 / 3
+ assert metrics["total_examples"] == 3
+
+ # New CI key present
+ assert "accuracy_ci" in metrics
+ ci = metrics["accuracy_ci"]
+ assert ci["sample_size"] == 3
+ assert 0.0 <= ci["lower"] <= ci["upper"] <= 1.0
+ # CI should contain the point estimate
+ assert ci["lower"] <= metrics["accuracy"] <= ci["upper"]
+
+ def test_category_ci_present(self, tmp_path):
+ from local_deep_research.benchmarks.metrics.calculation import (
+ calculate_metrics,
+ )
+
+ results_file = tmp_path / "results.jsonl"
+ results = [
+ {"is_correct": True, "category": "science"},
+ {"is_correct": False, "category": "science"},
+ {"is_correct": True, "category": "history"},
+ ]
+ with open(results_file, "w") as f:
+ for r in results:
+ f.write(json.dumps(r) + "\n")
+
+ metrics = calculate_metrics(str(results_file))
+
+ assert "accuracy_ci" in metrics["categories"]["science"]
+ assert "accuracy_ci" in metrics["categories"]["history"]
+
+ def test_empty_results_no_ci_error(self, tmp_path):
+ """Empty file should return error dict, not crash on CI."""
+ from local_deep_research.benchmarks.metrics.calculation import (
+ calculate_metrics,
+ )
+
+ results_file = tmp_path / "empty.jsonl"
+ results_file.write_text("")
+
+ metrics = calculate_metrics(str(results_file))
+ assert "error" in metrics