Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

chore(evals): remove dead and test-only eval helpers (#4079)

Tejas Choprade8969267c7d
diff --git a/headroom/evals/comprehensive_benchmark.py b/headroom/evals/comprehensive_benchmark.py
index 513ea2f36..13c7700e6 100644
--- a/headroom/evals/comprehensive_benchmark.py
+++ b/headroom/evals/comprehensive_benchmark.py
@@ -46,29 +46,6 @@ DEFAULT_TASKS = [
"gsm8k", # Math reasoning - generation-based, works with chat APIs
]
-# Tier 1 standard benchmarks for the suite runner
-TIER1_TASKS = [
- "gsm8k", # Math reasoning
- "truthfulqa_gen", # Factual accuracy
- "mmlu", # General knowledge (57 subjects)
- "arc_challenge", # Science reasoning
-]
-
-# Code tasks (need --confirm_run_unsafe_code)
-TIER1_CODE_TASKS = [
- "humaneval", # Code generation (pass@1)
-]
-
-EXTENDED_TASKS = [
- "mmlu", # 57 subjects - comprehensive but slow
- "gsm8k", # Math - requires multi-step reasoning
- "arc_challenge", # Harder science reasoning
-]
-
-TIER3_TASKS = [
- "hellaswag", # Commonsense reasoning
-]
-
@dataclass
class BenchmarkResult:
diff --git a/headroom/evals/datasets.py b/headroom/evals/datasets.py
index 9c1673efc..9f67fc694 100644
--- a/headroom/evals/datasets.py
+++ b/headroom/evals/datasets.py
@@ -1177,47 +1177,6 @@ def load_custom_dataset(path: Path | str) -> EvalSuite:
return EvalSuite.from_jsonl(path)
-def generate_retrieval_probes(
- context: str,
- n_probes: int = 5,
-) -> list[str]:
- """Generate retrieval probes from a context.
-
- Extracts key facts/entities that should be retrievable
- after compression.
-
- Args:
- context: The context to analyze
- n_probes: Number of probes to generate
-
- Returns:
- List of fact strings to probe for
- """
- import re
-
- probes = []
-
- # Look for specific patterns
- patterns = [
- r"\b[A-Z][a-z]+ [A-Z][a-z]+\b", # Names (e.g., "John Smith")
- r"\b\d{4}-\d{2}-\d{2}\b", # Dates (e.g., "2024-01-15")
- r"\b[A-Z]{2,}\b", # Acronyms (e.g., "API", "HTTP")
- r"\b\d+\.\d+%?\b", # Numbers (e.g., "99.9%", "123.45")
- r'"[^"]{5,50}"', # Quoted strings
- r"\b[a-z_]+_[a-z_]+\b", # Snake case identifiers
- ]
-
- for pattern in patterns:
- matches = re.findall(pattern, context)
- for match in matches[:2]: # Take up to 2 per pattern
- if match not in probes:
- probes.append(match.strip('"'))
- if len(probes) >= n_probes:
- return probes
-
- return probes
-
-
# =============================================================================
# DATASET REGISTRY & UTILITIES
# =============================================================================
diff --git a/headroom/evals/memory/judge.py b/headroom/evals/memory/judge.py
index 9713b7176..85091c9da 100644
--- a/headroom/evals/memory/judge.py
+++ b/headroom/evals/memory/judge.py
@@ -248,32 +248,6 @@ def _parse_judge_response(text: str) -> tuple[float, str]:
return score, reasoning
-def create_batch_judge(
- judge_fn: Callable[[str, str, str], tuple[float, str]],
- max_concurrent: int = 5,
-) -> Callable[[list[tuple[str, str, str]]], list[tuple[float, str]]]:
- """Create a batch judge function for parallel evaluation.
-
- Args:
- judge_fn: Single-item judge function.
- max_concurrent: Maximum concurrent API calls.
-
- Returns:
- A function that takes a list of (question, ground_truth, prediction)
- and returns a list of (score, reasoning).
- """
- from concurrent.futures import ThreadPoolExecutor
-
- def batch_judge(
- items: list[tuple[str, str, str]],
- ) -> list[tuple[float, str]]:
- with ThreadPoolExecutor(max_workers=max_concurrent) as executor:
- futures = [executor.submit(judge_fn, q, gt, pred) for q, gt, pred in items]
- return [f.result() for f in futures]
-
- return batch_judge
-
-
# Convenience function for simple scoring without LLM
def simple_judge(
question: str,
diff --git a/headroom/evals/metrics.py b/headroom/evals/metrics.py
index f2d96f881..8226fbee2 100644
--- a/headroom/evals/metrics.py
+++ b/headroom/evals/metrics.py
@@ -79,50 +79,6 @@ def compute_f1(response_a: str, response_b: str) -> float:
return 2 * precision * recall / (precision + recall)
-def compute_bleu(response_a: str, response_b: str, max_n: int = 4) -> float:
- """Compute BLEU-like score between two responses.
-
- Uses n-gram precision up to max_n.
- """
- tokens_a = tokenize(response_a)
- tokens_b = tokenize(response_b)
-
- if not tokens_a or not tokens_b:
- return 0.0
-
- precisions = []
-
- for n in range(1, max_n + 1):
- # Get n-grams
- ngrams_a = [tuple(tokens_a[i : i + n]) for i in range(len(tokens_a) - n + 1)]
- ngrams_b = [tuple(tokens_b[i : i + n]) for i in range(len(tokens_b) - n + 1)]
-
- if not ngrams_a:
- break
-
- counter_a = Counter(ngrams_a)
- counter_b = Counter(ngrams_b)
-
- # Clipped count
- clipped = sum((counter_a & counter_b).values())
- total = len(ngrams_a)
-
- if total > 0:
- precisions.append(clipped / total)
- else:
- precisions.append(0.0)
-
- if not precisions or all(p == 0 for p in precisions):
- return 0.0
-
- # Geometric mean of precisions
- import math
-
- nonzero_precisions = [p for p in precisions if p > 0]
- log_sum = sum(math.log(p) for p in nonzero_precisions) / len(nonzero_precisions)
- return math.exp(log_sum)
-
-
def compute_rouge_l(response_a: str, response_b: str) -> float:
"""Compute ROUGE-L score (longest common subsequence).
diff --git a/tests/test_evals_datasets.py b/tests/test_evals_datasets.py
index 9f116f365..1f42882d4 100644
--- a/tests/test_evals_datasets.py
+++ b/tests/test_evals_datasets.py
@@ -267,7 +267,7 @@ def test_load_bfcl_success_and_download_failure(monkeypatch: pytest.MonkeyPatch)
datasets.load_bfcl(category="parallel")
-def test_tool_output_samples_custom_dataset_and_probe_generation(tmp_path) -> None:
+def test_tool_output_samples_and_custom_dataset(tmp_path) -> None:
tool_outputs = datasets.load_tool_output_samples()
assert tool_outputs.name == "ToolOutputSamples"
assert len(tool_outputs.cases) >= 8
@@ -284,16 +284,6 @@ def test_tool_output_samples_custom_dataset_and_probe_generation(tmp_path) -> No
custom_suite = datasets.load_custom_dataset(custom_path)
assert custom_suite.cases[0].id == "case1"
- probes = datasets.generate_retrieval_probes(
- 'Alice Smith deployed API on 2024-01-15 at 99.9% confidence for "Launch Ready" and build_id',
- n_probes=5,
- )
- assert "Alice Smith" in probes
- assert "2024-01-15" in probes
- assert "API" in probes
- assert "99.9" in probes
- assert "Launch Ready" in probes
-
def test_dataset_registry_helpers(monkeypatch: pytest.MonkeyPatch) -> None:
categories = datasets.list_available_datasets()
diff --git a/tests/test_evals_metrics.py b/tests/test_evals_metrics.py
index 9d5432be0..5d34428f1 100644
--- a/tests/test_evals_metrics.py
+++ b/tests/test_evals_metrics.py
@@ -16,16 +16,11 @@ def test_normalize_tokenize_and_exact_match() -> None:
assert metrics.compute_exact_match("hello", "world") is False
-def test_f1_bleu_and_rouge_cover_edge_cases() -> None:
+def test_f1_and_rouge_cover_edge_cases() -> None:
assert metrics.compute_f1("", "value") == 0.0
assert metrics.compute_f1("alpha beta", "gamma delta") == 0.0
assert metrics.compute_f1("alpha beta gamma", "alpha gamma") == pytest.approx(0.8)
- assert metrics.compute_bleu("", "value") == 0.0
- assert metrics.compute_bleu("one", "one") == pytest.approx(1.0)
- assert metrics.compute_bleu("alpha beta", "gamma delta") == 0.0
- assert metrics.compute_bleu("alpha beta", "alpha beta gamma", max_n=4) == pytest.approx(1.0)
-
assert metrics.compute_rouge_l("", "value") == 0.0
assert metrics.compute_rouge_l("alpha beta", "gamma delta") == 0.0
assert metrics.compute_rouge_l("alpha beta gamma", "alpha gamma") == pytest.approx(0.8)