~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
feat(benchmarks): add statistical functions module (#4029)
Ishitta2808f0fa9dd7
diff --git a/src/local_deep_research/benchmarks/metrics/__init__.py b/src/local_deep_research/benchmarks/metrics/__init__.pyindex 50001183cf..db2b3efbfe 100644--- a/src/local_deep_research/benchmarks/metrics/__init__.py+++ b/src/local_deep_research/benchmarks/metrics/__init__.py@@ -13,6 +13,10 @@ from .calculation import (calculate_speed_metrics,)from .reporting import generate_report+from .statistics import (+ sample_size_for_difference,+ wilson_score_interval,+)__all__ = ["calculate_combined_score",@@ -21,4 +25,6 @@ __all__ = ["calculate_resource_metrics","calculate_speed_metrics","generate_report",+ "sample_size_for_difference",+ "wilson_score_interval",]diff --git a/src/local_deep_research/benchmarks/metrics/calculation.py b/src/local_deep_research/benchmarks/metrics/calculation.pyindex c2558c2724..f288acb3ef 100644--- a/src/local_deep_research/benchmarks/metrics/calculation.py+++ b/src/local_deep_research/benchmarks/metrics/calculation.py@@ -38,12 +38,17 @@ def calculate_metrics(results_file: str) -> Dict[str, Any]:if not results:return {"error": "No results found"}+ from .statistics import wilson_score_interval+# Calculate accuracygraded_results = [r for r in results if "is_correct" in r]correct_count = sum(1 for r in graded_results if r.get("is_correct", False))total_graded = len(graded_results)accuracy = correct_count / total_graded if total_graded else 0+ # Wilson score confidence interval for accuracy+ accuracy_ci = wilson_score_interval(correct_count, total_graded)+# Calculate average processing time if availableprocessing_times = [r.get("processing_time", 0) for r in results if "processing_time" in r@@ -77,6 +82,7 @@ def calculate_metrics(results_file: str) -> Dict[str, Any]:"graded_examples": total_graded,"correct": correct_count,"accuracy": accuracy,+ "accuracy_ci": accuracy_ci,"average_processing_time": avg_time,"average_confidence": avg_confidence,"error_count": error_count,@@ -98,13 +104,15 @@ def calculate_metrics(results_file: str) -> Dict[str, Any]:if categories:category_metrics = {}for category, counts in categories.items():+ cat_accuracy = (+ counts["correct"] / counts["total"] if counts["total"] else 0+ )category_metrics[category] = {"total": counts["total"],"correct": counts["correct"],- "accuracy": (- counts["correct"] / counts["total"]- if counts["total"]- else 0+ "accuracy": cat_accuracy,+ "accuracy_ci": wilson_score_interval(+ counts["correct"], counts["total"]),}metrics["categories"] = category_metricsdiff --git a/src/local_deep_research/benchmarks/metrics/reporting.py b/src/local_deep_research/benchmarks/metrics/reporting.pyindex 39051d091e..e40e842a00 100644--- a/src/local_deep_research/benchmarks/metrics/reporting.py+++ b/src/local_deep_research/benchmarks/metrics/reporting.py@@ -65,6 +65,14 @@ def generate_report(f"- **Accuracy**: {metrics.get('accuracy', 0):.3f}",]+ # Add confidence interval if available+ accuracy_ci = metrics.get("accuracy_ci")+ if accuracy_ci and accuracy_ci.get("sample_size", 0) > 0:+ report.append(+ f"- **95% CI**: [{accuracy_ci['lower']:.3f}, {accuracy_ci['upper']:.3f}] "+ f"(Wilson score, n={accuracy_ci['sample_size']})"+ )+if "average_processing_time" in metrics:report.append(f"- **Average Processing Time**: {metrics['average_processing_time']:.2f} seconds"@@ -91,6 +99,11 @@ def generate_report(report.append(f"- **Total**: {category_metrics['total']}")report.append(f"- **Correct**: {category_metrics['correct']}")report.append(f"- **Accuracy**: {category_metrics['accuracy']:.3f}")+ cat_ci = category_metrics.get("accuracy_ci")+ if cat_ci and cat_ci.get("sample_size", 0) > 0:+ report.append(+ f"- **95% CI**: [{cat_ci['lower']:.3f}, {cat_ci['upper']:.3f}]"+ )report.append("")# Add configuration info if provideddiff --git a/src/local_deep_research/benchmarks/metrics/statistics.py b/src/local_deep_research/benchmarks/metrics/statistics.pynew file mode 100644index 0000000000..d58ce3bfdd--- /dev/null+++ b/src/local_deep_research/benchmarks/metrics/statistics.py@@ -0,0 +1,211 @@+"""+Statistical functions for benchmark evaluation.++Provides confidence intervals, sample size calculations, and+proportion statistics for interpreting benchmark results.+All implementations are pure Python.+"""++import math+++def normal_quantile(p: float) -> float:+ """+ Inverse normal CDF (percent-point function).++ Uses the Beasley-Springer-Moro rational approximation,+ accurate to approximately 1e-8.++ Args:+ p: Probability in (0, 1).++ Returns:+ z such that P(Z <= z) = p for standard normal Z.++ Raises:+ ValueError: If p is not in (0, 1).+ """+ if p <= 0 or p >= 1:+ raise ValueError(f"p must be in (0, 1), got {p}")++ # Rational approximation coefficients (Beasley-Springer-Moro)+ a = [+ -3.969683028665376e01,+ 2.209460984245205e02,+ -2.759285104469687e02,+ 1.383577518672690e02,+ -3.066479806614716e01,+ 2.506628277459239e00,+ ]+ b = [+ -5.447609879822406e01,+ 1.615858368580409e02,+ -1.556989798598866e02,+ 6.680131188771972e01,+ -1.328068155288572e01,+ ]+ c = [+ -7.784894002430293e-03,+ -3.223964580411365e-01,+ -2.400758277161838e00,+ -2.549732539343734e00,+ 4.374664141464968e00,+ 2.938163982698783e00,+ ]+ d = [+ 7.784695709041462e-03,+ 3.224671290700398e-01,+ 2.445134137142996e00,+ 3.754408661907416e00,+ ]++ p_low = 0.02425+ p_high = 1 - p_low++ if p < p_low:+ # Lower tail+ q = math.sqrt(-2 * math.log(p))+ return (+ ((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]+ ) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1)+ if p <= p_high:+ # Central region+ q = p - 0.5+ r = q * q+ return (+ (+ ((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r+ + a[5]+ )+ * q+ ) / (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1)+ # Upper tail+ q = math.sqrt(-2 * math.log(1 - p))+ return -(+ ((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]+ ) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1)+++def wilson_score_interval(+ successes: int,+ total: int,+ confidence: float = 0.95,+) -> dict:+ """+ Wilson score confidence interval for a binomial proportion.++ Unlike the normal (Wald) approximation, the Wilson interval:+ - Never produces bounds outside [0, 1]+ - Behaves correctly at 0% and 100% accuracy+ - Has better coverage at small sample sizes++ Args:+ successes: Number of successes (correct answers).+ total: Total number of trials (graded examples).+ confidence: Confidence level (default 0.95 for 95% CI).++ Returns:+ Dictionary with keys:+ - lower: Lower bound of the interval+ - upper: Upper bound of the interval+ - center: Center of the Wilson interval (not the raw proportion)+ - margin_of_error: Half-width of the interval+ - sample_size: The total n used+ """+ if total <= 0:+ return {+ "lower": 0.0,+ "upper": 0.0,+ "center": 0.0,+ "margin_of_error": 0.0,+ "sample_size": 0,+ }++ if successes < 0 or successes > total:+ raise ValueError(+ f"successes must be in [0, total], got successes={successes}, total={total}"+ )++ p_hat = successes / total+ z = normal_quantile(1 - (1 - confidence) / 2)+ z2 = z * z++ denominator = 1 + z2 / total+ center = (p_hat + z2 / (2 * total)) / denominator+ half_width = (+ z * math.sqrt(p_hat * (1 - p_hat) / total + z2 / (4 * total * total))+ ) / denominator++ lower = max(0.0, center - half_width)+ upper = min(1.0, center + half_width)++ return {+ "lower": lower,+ "upper": upper,+ "center": center,+ "margin_of_error": half_width,+ "sample_size": total,+ }+++def proportion_std_error(p: float, n: int) -> float:+ """+ Standard error of a sample proportion.++ Args:+ p: Observed proportion (0 to 1).+ n: Sample size.++ Returns:+ Standard error, or 0.0 if n <= 0.+ """+ if n <= 0:+ return 0.0+ if p < 0 or p > 1:+ raise ValueError(f"p must be in [0, 1], got {p}")+ return math.sqrt(p * (1 - p) / n)+++def sample_size_for_difference(+ p1: float,+ p2: float,+ power: float = 0.8,+ alpha: float = 0.05,+) -> int:+ """+ Required sample size per group to detect a difference between+ two independent proportions.++ Uses the formula for a two-sided two-proportion z-test:+ n = (z_alpha/2 + z_beta)^2 * (p1(1-p1) + p2(1-p2)) / (p1 - p2)^2++ Args:+ p1: Expected proportion for group 1.+ p2: Expected proportion for group 2.+ power: Statistical power (default 0.8 = 80%).+ alpha: Significance level (default 0.05 for two-sided test).++ Returns:+ Required sample size per group (rounded up).++ Raises:+ ValueError: If p1 == p2 (no difference to detect).+ """+ if p1 == p2:+ raise ValueError(+ "p1 and p2 must differ to compute required sample size"+ )+ for name, val in [("p1", p1), ("p2", p2)]:+ if val < 0 or val > 1:+ raise ValueError(f"{name} must be in [0, 1], got {val}")+ for name, val in [("power", power), ("alpha", alpha)]:+ if val <= 0 or val >= 1:+ raise ValueError(f"{name} must be in (0, 1), got {val}")++ z_alpha = normal_quantile(1 - alpha / 2)+ z_beta = normal_quantile(power)++ numerator = (z_alpha + z_beta) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2))+ denominator = (p1 - p2) ** 2++ return math.ceil(numerator / denominator)diff --git a/src/local_deep_research/benchmarks/web_api/benchmark_service.py b/src/local_deep_research/benchmarks/web_api/benchmark_service.pyindex e0f56c09e1..f70db309db 100644--- a/src/local_deep_research/benchmarks/web_api/benchmark_service.py+++ b/src/local_deep_research/benchmarks/web_api/benchmark_service.py@@ -1362,21 +1362,19 @@ class BenchmarkService:f"estimated_remaining: {estimated_time_remaining:.2f}s")- # Calculate accuracy confidence interval (95% confidence)+ # Calculate accuracy confidence interval (Wilson score, 95%)if results and len(results) >= 3:- import math+ from local_deep_research.benchmarks.metrics.statistics import (+ wilson_score_interval,+ )n = len(results)- p = running_accuracy / 100 if running_accuracy else 0- # Standard error for proportion- se = math.sqrt(p * (1 - p) / n)- # 95% confidence interval (±1.96 * SE)- margin_of_error = 1.96 * se * 100- _running_acc = running_accuracy or 0.0+ correct = sum(1 for r in results if r.is_correct)+ ci = wilson_score_interval(correct, n)accuracy_confidence = {- "lower_bound": max(0, _running_acc - margin_of_error),- "upper_bound": min(100, _running_acc + margin_of_error),- "margin_of_error": margin_of_error,+ "lower_bound": ci["lower"] * 100,+ "upper_bound": ci["upper"] * 100,+ "margin_of_error": ci["margin_of_error"] * 100,"sample_size": n,}diff --git a/tests/benchmarks/test_statistics.py b/tests/benchmarks/test_statistics.pynew file mode 100644index 0000000000..d1b56d007f--- /dev/null+++ b/tests/benchmarks/test_statistics.py@@ -0,0 +1,364 @@+"""+Tests for benchmarks/metrics/statistics.py++Tests cover:+- normal_quantile inverse CDF+- wilson_score_interval confidence intervals+- proportion_std_error+- sample_size_for_difference power analysis+- Integration: calculate_metrics returns accuracy_ci+"""++import json++import pytest+++class TestNormalQuantile:+ """Tests for the normal_quantile function."""++ def test_median(self):+ from local_deep_research.benchmarks.metrics.statistics import (+ normal_quantile,+ )++ assert abs(normal_quantile(0.5) - 0.0) < 1e-6++ def test_975_gives_196(self):+ from local_deep_research.benchmarks.metrics.statistics import (+ normal_quantile,+ )++ assert abs(normal_quantile(0.975) - 1.96) < 0.001++ def test_025_gives_negative_196(self):+ from local_deep_research.benchmarks.metrics.statistics import (+ normal_quantile,+ )++ assert abs(normal_quantile(0.025) - (-1.96)) < 0.001++ def test_symmetry(self):+ from local_deep_research.benchmarks.metrics.statistics import (+ normal_quantile,+ )++ for p in [0.01, 0.05, 0.1, 0.25]:+ assert abs(normal_quantile(p) + normal_quantile(1 - p)) < 1e-6++ def test_raises_on_zero(self):+ from local_deep_research.benchmarks.metrics.statistics import (+ normal_quantile,+ )++ with pytest.raises(ValueError):+ normal_quantile(0.0)++ def test_raises_on_one(self):+ from local_deep_research.benchmarks.metrics.statistics import (+ normal_quantile,+ )++ with pytest.raises(ValueError):+ normal_quantile(1.0)++ def test_raises_on_negative(self):+ from local_deep_research.benchmarks.metrics.statistics import (+ normal_quantile,+ )++ with pytest.raises(ValueError):+ normal_quantile(-0.5)++ def test_extreme_tails(self):+ from local_deep_research.benchmarks.metrics.statistics import (+ normal_quantile,+ )++ # p=0.001 -> z ~ -3.09+ assert abs(normal_quantile(0.001) - (-3.09)) < 0.01+ # p=0.999 -> z ~ 3.09+ assert abs(normal_quantile(0.999) - 3.09) < 0.01+++class TestWilsonScoreInterval:+ """Tests for the wilson_score_interval function."""++ def test_91_of_100(self):+ """91/100 at 95% should give approximately [0.837, 0.955]."""+ from local_deep_research.benchmarks.metrics.statistics import (+ wilson_score_interval,+ )++ ci = wilson_score_interval(91, 100)+ assert abs(ci["lower"] - 0.837) < 0.005+ assert abs(ci["upper"] - 0.955) < 0.005+ assert ci["sample_size"] == 100++ def test_perfect_score(self):+ """100/100 should NOT give [1.0, 1.0] (unlike normal approx)."""+ from local_deep_research.benchmarks.metrics.statistics import (+ wilson_score_interval,+ )++ ci = wilson_score_interval(100, 100)+ assert ci["upper"] == 1.0+ assert ci["lower"] < 1.0 # Wilson correctly shows uncertainty+ assert ci["lower"] > 0.95++ def test_zero_score(self):+ """0/100 should NOT give [0.0, 0.0] (unlike normal approx)."""+ from local_deep_research.benchmarks.metrics.statistics import (+ wilson_score_interval,+ )++ ci = wilson_score_interval(0, 100)+ assert ci["lower"] == 0.0+ assert ci["upper"] > 0.0 # Wilson correctly shows uncertainty+ assert ci["upper"] < 0.05++ def test_zero_total(self):+ """0/0 should return zeros without error."""+ from local_deep_research.benchmarks.metrics.statistics import (+ wilson_score_interval,+ )++ ci = wilson_score_interval(0, 0)+ assert ci["lower"] == 0.0+ assert ci["upper"] == 0.0+ assert ci["margin_of_error"] == 0.0+ assert ci["sample_size"] == 0++ def test_raises_on_successes_greater_than_total(self):+ """successes > total should raise ValueError."""+ from local_deep_research.benchmarks.metrics.statistics import (+ wilson_score_interval,+ )++ with pytest.raises(ValueError, match="successes must be in"):+ wilson_score_interval(110, 100)++ def test_raises_on_negative_successes(self):+ """Negative successes should raise ValueError."""+ from local_deep_research.benchmarks.metrics.statistics import (+ wilson_score_interval,+ )++ with pytest.raises(ValueError, match="successes must be in"):+ wilson_score_interval(-1, 100)++ def test_single_observation(self):+ """1/1 should give a wide interval."""+ from local_deep_research.benchmarks.metrics.statistics import (+ wilson_score_interval,+ )++ ci = wilson_score_interval(1, 1)+ assert ci["lower"] < 0.5+ assert ci["upper"] == 1.0++ def test_bounds_within_0_1(self):+ """Bounds should always be in [0, 1]."""+ from local_deep_research.benchmarks.metrics.statistics import (+ wilson_score_interval,+ )++ for successes, total in [+ (0, 10),+ (10, 10),+ (5, 10),+ (1, 100),+ (99, 100),+ ]:+ ci = wilson_score_interval(successes, total)+ assert 0.0 <= ci["lower"] <= ci["upper"] <= 1.0++ def test_higher_n_narrower_interval(self):+ """Larger sample size should give narrower CI at same proportion."""+ from local_deep_research.benchmarks.metrics.statistics import (+ wilson_score_interval,+ )++ ci_small = wilson_score_interval(9, 10)+ ci_large = wilson_score_interval(90, 100)+ assert ci_large["margin_of_error"] < ci_small["margin_of_error"]++ def test_90_confidence_narrower_than_99(self):+ """90% CI should be narrower than 99% CI."""+ from local_deep_research.benchmarks.metrics.statistics import (+ wilson_score_interval,+ )++ ci_90 = wilson_score_interval(50, 100, confidence=0.90)+ ci_99 = wilson_score_interval(50, 100, confidence=0.99)+ assert ci_90["margin_of_error"] < ci_99["margin_of_error"]+++class TestProportionStdError:+ """Tests for the proportion_std_error function."""++ def test_basic(self):+ from local_deep_research.benchmarks.metrics.statistics import (+ proportion_std_error,+ )++ # p=0.5, n=100 -> sqrt(0.25/100) = 0.05+ assert abs(proportion_std_error(0.5, 100) - 0.05) < 1e-10++ def test_zero_n(self):+ from local_deep_research.benchmarks.metrics.statistics import (+ proportion_std_error,+ )++ assert proportion_std_error(0.5, 0) == 0.0++ def test_extreme_proportions(self):+ from local_deep_research.benchmarks.metrics.statistics import (+ proportion_std_error,+ )++ assert proportion_std_error(0.0, 100) == 0.0+ assert proportion_std_error(1.0, 100) == 0.0++ def test_raises_on_invalid_p(self):+ from local_deep_research.benchmarks.metrics.statistics import (+ proportion_std_error,+ )++ with pytest.raises(ValueError, match="p must be in"):+ proportion_std_error(-0.1, 100)+ with pytest.raises(ValueError, match="p must be in"):+ proportion_std_error(1.5, 100)+++class TestSampleSizeForDifference:+ """Tests for the sample_size_for_difference function."""++ def test_5pp_difference(self):+ """5pp difference (85% vs 90%) should need ~680 per group."""+ from local_deep_research.benchmarks.metrics.statistics import (+ sample_size_for_difference,+ )++ n = sample_size_for_difference(0.85, 0.90)+ assert 600 <= n <= 760++ def test_10pp_difference(self):+ """10pp difference (80% vs 90%) should need ~200 per group."""+ from local_deep_research.benchmarks.metrics.statistics import (+ sample_size_for_difference,+ )++ n = sample_size_for_difference(0.80, 0.90)+ assert 170 <= n <= 230++ def test_15pp_difference(self):+ """15pp difference (75% vs 90%) should need ~90 per group."""+ from local_deep_research.benchmarks.metrics.statistics import (+ sample_size_for_difference,+ )++ n = sample_size_for_difference(0.75, 0.90)+ assert 75 <= n <= 105++ def test_raises_on_equal_proportions(self):+ from local_deep_research.benchmarks.metrics.statistics import (+ sample_size_for_difference,+ )++ with pytest.raises(ValueError):+ sample_size_for_difference(0.9, 0.9)++ def test_raises_on_invalid_proportions(self):+ from local_deep_research.benchmarks.metrics.statistics import (+ sample_size_for_difference,+ )++ with pytest.raises(ValueError, match="p1 must be in"):+ sample_size_for_difference(-0.1, 0.9)+ with pytest.raises(ValueError, match="p2 must be in"):+ sample_size_for_difference(0.8, 1.5)++ def test_raises_on_invalid_power_or_alpha(self):+ from local_deep_research.benchmarks.metrics.statistics import (+ sample_size_for_difference,+ )++ with pytest.raises(ValueError, match="power must be in"):+ sample_size_for_difference(0.8, 0.9, power=0.0)+ with pytest.raises(ValueError, match="alpha must be in"):+ sample_size_for_difference(0.8, 0.9, alpha=1.0)++ def test_higher_power_needs_more_samples(self):+ from local_deep_research.benchmarks.metrics.statistics import (+ sample_size_for_difference,+ )++ n_80 = sample_size_for_difference(0.80, 0.90, power=0.80)+ n_95 = sample_size_for_difference(0.80, 0.90, power=0.95)+ assert n_95 > n_80+++class TestCalculateMetricsIntegration:+ """Test that calculate_metrics() returns accuracy_ci (backward compat)."""++ def test_accuracy_ci_present(self, tmp_path):+ from local_deep_research.benchmarks.metrics.calculation import (+ calculate_metrics,+ )++ results_file = tmp_path / "results.jsonl"+ results = [+ {"is_correct": True, "processing_time": 1.5},+ {"is_correct": True, "processing_time": 2.0},+ {"is_correct": False, "processing_time": 1.0},+ ]+ with open(results_file, "w") as f:+ for r in results:+ f.write(json.dumps(r) + "\n")++ metrics = calculate_metrics(str(results_file))++ # Existing keys still present+ assert metrics["accuracy"] == 2 / 3+ assert metrics["total_examples"] == 3++ # New CI key present+ assert "accuracy_ci" in metrics+ ci = metrics["accuracy_ci"]+ assert ci["sample_size"] == 3+ assert 0.0 <= ci["lower"] <= ci["upper"] <= 1.0+ # CI should contain the point estimate+ assert ci["lower"] <= metrics["accuracy"] <= ci["upper"]++ def test_category_ci_present(self, tmp_path):+ from local_deep_research.benchmarks.metrics.calculation import (+ calculate_metrics,+ )++ results_file = tmp_path / "results.jsonl"+ results = [+ {"is_correct": True, "category": "science"},+ {"is_correct": False, "category": "science"},+ {"is_correct": True, "category": "history"},+ ]+ with open(results_file, "w") as f:+ for r in results:+ f.write(json.dumps(r) + "\n")++ metrics = calculate_metrics(str(results_file))++ assert "accuracy_ci" in metrics["categories"]["science"]+ assert "accuracy_ci" in metrics["categories"]["history"]++ def test_empty_results_no_ci_error(self, tmp_path):+ """Empty file should return error dict, not crash on CI."""+ from local_deep_research.benchmarks.metrics.calculation import (+ calculate_metrics,+ )++ results_file = tmp_path / "empty.jsonl"+ results_file.write_text("")++ metrics = calculate_metrics(str(results_file))+ assert "error" in metrics