Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
chore(evals): remove dead and test-only eval helpers (#4079)
Tejas Choprade8969267c7d
diff --git a/headroom/evals/comprehensive_benchmark.py b/headroom/evals/comprehensive_benchmark.pyindex 513ea2f36..13c7700e6 100644--- a/headroom/evals/comprehensive_benchmark.py+++ b/headroom/evals/comprehensive_benchmark.py@@ -46,29 +46,6 @@ DEFAULT_TASKS = ["gsm8k", # Math reasoning - generation-based, works with chat APIs]-# Tier 1 standard benchmarks for the suite runner-TIER1_TASKS = [- "gsm8k", # Math reasoning- "truthfulqa_gen", # Factual accuracy- "mmlu", # General knowledge (57 subjects)- "arc_challenge", # Science reasoning-]--# Code tasks (need --confirm_run_unsafe_code)-TIER1_CODE_TASKS = [- "humaneval", # Code generation (pass@1)-]--EXTENDED_TASKS = [- "mmlu", # 57 subjects - comprehensive but slow- "gsm8k", # Math - requires multi-step reasoning- "arc_challenge", # Harder science reasoning-]--TIER3_TASKS = [- "hellaswag", # Commonsense reasoning-]-@dataclassclass BenchmarkResult:diff --git a/headroom/evals/datasets.py b/headroom/evals/datasets.pyindex 9c1673efc..9f67fc694 100644--- a/headroom/evals/datasets.py+++ b/headroom/evals/datasets.py@@ -1177,47 +1177,6 @@ def load_custom_dataset(path: Path | str) -> EvalSuite:return EvalSuite.from_jsonl(path)-def generate_retrieval_probes(- context: str,- n_probes: int = 5,-) -> list[str]:- """Generate retrieval probes from a context.-- Extracts key facts/entities that should be retrievable- after compression.-- Args:- context: The context to analyze- n_probes: Number of probes to generate-- Returns:- List of fact strings to probe for- """- import re-- probes = []-- # Look for specific patterns- patterns = [- r"\b[A-Z][a-z]+ [A-Z][a-z]+\b", # Names (e.g., "John Smith")- r"\b\d{4}-\d{2}-\d{2}\b", # Dates (e.g., "2024-01-15")- r"\b[A-Z]{2,}\b", # Acronyms (e.g., "API", "HTTP")- r"\b\d+\.\d+%?\b", # Numbers (e.g., "99.9%", "123.45")- r'"[^"]{5,50}"', # Quoted strings- r"\b[a-z_]+_[a-z_]+\b", # Snake case identifiers- ]-- for pattern in patterns:- matches = re.findall(pattern, context)- for match in matches[:2]: # Take up to 2 per pattern- if match not in probes:- probes.append(match.strip('"'))- if len(probes) >= n_probes:- return probes-- return probes--# =============================================================================# DATASET REGISTRY & UTILITIES# =============================================================================diff --git a/headroom/evals/memory/judge.py b/headroom/evals/memory/judge.pyindex 9713b7176..85091c9da 100644--- a/headroom/evals/memory/judge.py+++ b/headroom/evals/memory/judge.py@@ -248,32 +248,6 @@ def _parse_judge_response(text: str) -> tuple[float, str]:return score, reasoning-def create_batch_judge(- judge_fn: Callable[[str, str, str], tuple[float, str]],- max_concurrent: int = 5,-) -> Callable[[list[tuple[str, str, str]]], list[tuple[float, str]]]:- """Create a batch judge function for parallel evaluation.-- Args:- judge_fn: Single-item judge function.- max_concurrent: Maximum concurrent API calls.-- Returns:- A function that takes a list of (question, ground_truth, prediction)- and returns a list of (score, reasoning).- """- from concurrent.futures import ThreadPoolExecutor-- def batch_judge(- items: list[tuple[str, str, str]],- ) -> list[tuple[float, str]]:- with ThreadPoolExecutor(max_workers=max_concurrent) as executor:- futures = [executor.submit(judge_fn, q, gt, pred) for q, gt, pred in items]- return [f.result() for f in futures]-- return batch_judge--# Convenience function for simple scoring without LLMdef simple_judge(question: str,diff --git a/headroom/evals/metrics.py b/headroom/evals/metrics.pyindex f2d96f881..8226fbee2 100644--- a/headroom/evals/metrics.py+++ b/headroom/evals/metrics.py@@ -79,50 +79,6 @@ def compute_f1(response_a: str, response_b: str) -> float:return 2 * precision * recall / (precision + recall)-def compute_bleu(response_a: str, response_b: str, max_n: int = 4) -> float:- """Compute BLEU-like score between two responses.-- Uses n-gram precision up to max_n.- """- tokens_a = tokenize(response_a)- tokens_b = tokenize(response_b)-- if not tokens_a or not tokens_b:- return 0.0-- precisions = []-- for n in range(1, max_n + 1):- # Get n-grams- ngrams_a = [tuple(tokens_a[i : i + n]) for i in range(len(tokens_a) - n + 1)]- ngrams_b = [tuple(tokens_b[i : i + n]) for i in range(len(tokens_b) - n + 1)]-- if not ngrams_a:- break-- counter_a = Counter(ngrams_a)- counter_b = Counter(ngrams_b)-- # Clipped count- clipped = sum((counter_a & counter_b).values())- total = len(ngrams_a)-- if total > 0:- precisions.append(clipped / total)- else:- precisions.append(0.0)-- if not precisions or all(p == 0 for p in precisions):- return 0.0-- # Geometric mean of precisions- import math-- nonzero_precisions = [p for p in precisions if p > 0]- log_sum = sum(math.log(p) for p in nonzero_precisions) / len(nonzero_precisions)- return math.exp(log_sum)--def compute_rouge_l(response_a: str, response_b: str) -> float:"""Compute ROUGE-L score (longest common subsequence).diff --git a/tests/test_evals_datasets.py b/tests/test_evals_datasets.pyindex 9f116f365..1f42882d4 100644--- a/tests/test_evals_datasets.py+++ b/tests/test_evals_datasets.py@@ -267,7 +267,7 @@ def test_load_bfcl_success_and_download_failure(monkeypatch: pytest.MonkeyPatch)datasets.load_bfcl(category="parallel")-def test_tool_output_samples_custom_dataset_and_probe_generation(tmp_path) -> None:+def test_tool_output_samples_and_custom_dataset(tmp_path) -> None:tool_outputs = datasets.load_tool_output_samples()assert tool_outputs.name == "ToolOutputSamples"assert len(tool_outputs.cases) >= 8@@ -284,16 +284,6 @@ def test_tool_output_samples_custom_dataset_and_probe_generation(tmp_path) -> Nocustom_suite = datasets.load_custom_dataset(custom_path)assert custom_suite.cases[0].id == "case1"- probes = datasets.generate_retrieval_probes(- 'Alice Smith deployed API on 2024-01-15 at 99.9% confidence for "Launch Ready" and build_id',- n_probes=5,- )- assert "Alice Smith" in probes- assert "2024-01-15" in probes- assert "API" in probes- assert "99.9" in probes- assert "Launch Ready" in probes-def test_dataset_registry_helpers(monkeypatch: pytest.MonkeyPatch) -> None:categories = datasets.list_available_datasets()diff --git a/tests/test_evals_metrics.py b/tests/test_evals_metrics.pyindex 9d5432be0..5d34428f1 100644--- a/tests/test_evals_metrics.py+++ b/tests/test_evals_metrics.py@@ -16,16 +16,11 @@ def test_normalize_tokenize_and_exact_match() -> None:assert metrics.compute_exact_match("hello", "world") is False-def test_f1_bleu_and_rouge_cover_edge_cases() -> None:+def test_f1_and_rouge_cover_edge_cases() -> None:assert metrics.compute_f1("", "value") == 0.0assert metrics.compute_f1("alpha beta", "gamma delta") == 0.0assert metrics.compute_f1("alpha beta gamma", "alpha gamma") == pytest.approx(0.8)- assert metrics.compute_bleu("", "value") == 0.0- assert metrics.compute_bleu("one", "one") == pytest.approx(1.0)- assert metrics.compute_bleu("alpha beta", "gamma delta") == 0.0- assert metrics.compute_bleu("alpha beta", "alpha beta gamma", max_n=4) == pytest.approx(1.0)-assert metrics.compute_rouge_l("", "value") == 0.0assert metrics.compute_rouge_l("alpha beta", "gamma delta") == 0.0assert metrics.compute_rouge_l("alpha beta gamma", "alpha gamma") == pytest.approx(0.8)