Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(evals): add zero-cost tool schema compaction integrity eval (#817)
Ashish53a08c63bf56
diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.ymlindex 20e24a6e4..bbaef6d4c 100644--- a/.github/workflows/eval.yml+++ b/.github/workflows/eval.yml@@ -54,6 +54,17 @@ jobs:print(f'CCR Round-trip: {result.passed_cases}/{result.total_cases} passed')assert result.passed, f'CCR failures: {result.errors}'"++ - name: Run tool schema compaction integrity eval (zero cost)+ run: |+ python -c "+ from headroom.evals.runners.compression_only import CompressionOnlyRunner+ runner = CompressionOnlyRunner()+ result = runner.evaluate_tool_schema_compaction()+ print(f'Tool schema compaction: {result.passed_cases}/{result.total_cases} passed, {result.total_tokens_saved} annotation tokens stripped')+ assert result.passed, f'Schema compaction failures: {result.errors}'+ "+# OPENAI_API_KEY is intentionally not set in the public OSS repo# (the secret list is empty). The CCR round-trip step above is the# mandatory gate; this step only runs when an operator has wireddiff --git a/headroom/evals/runners/compression_only.py b/headroom/evals/runners/compression_only.pyindex b999ab116..4ef71dbc0 100644--- a/headroom/evals/runners/compression_only.py+++ b/headroom/evals/runners/compression_only.py@@ -5,6 +5,7 @@ Used for:- CCR lossless round-trip verification- Information retention (probe facts survive compression)- Needle retention (specific values preserved in compressed output)+- Tool schema compaction integrity (property names survive annotation stripping)"""from __future__ import annotations@@ -335,3 +336,275 @@ class CompressionOnlyRunner:)return cases[:n]++ def generate_tool_schema_cases(self) -> list[dict[str, Any]]:+ """Generate tool schema test cases for compaction integrity verification.++ Each case exercises a different way a DROP_KEY can appear as a+ property *name* inside a JSON Schema `properties` object.+ The cases also include annotation keys at the schema level so we+ can verify those ARE still stripped.+ """+ return [+ {+ "id": "schema_title_property",+ "description": "property named 'title' must survive; schema-level title must be dropped",+ "payload": {+ "tools": [+ {+ "type": "function",+ "name": "eval_cells",+ "description": "Evaluate notebook cells.",+ "parameters": {+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "title": "EvalCellsParameters",+ "type": "object",+ "properties": {+ "cells": {+ "type": "array",+ "items": {+ "type": "object",+ "title": "CellItem",+ "properties": {+ "language": {"type": "string"},+ "code": {"type": "string"},+ "title": {"type": "string"},+ },+ "required": ["language", "code", "title"],+ },+ }+ },+ "required": ["cells"],+ },+ }+ ]+ },+ "must_preserve": ["title"],+ "must_drop_schema_annotations": True,+ },+ {+ "id": "schema_deprecated_property",+ "description": "property named 'deprecated' must survive",+ "payload": {+ "tools": [+ {+ "type": "function",+ "name": "list_apis",+ "description": "List available APIs with their status.",+ "parameters": {+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "title": "ListApisParameters",+ "type": "object",+ "properties": {+ "deprecated": {+ "type": "boolean",+ "description": "Include deprecated APIs in results.",+ },+ "name": {"type": "string"},+ },+ "required": ["deprecated"],+ },+ }+ ]+ },+ "must_preserve": ["deprecated"],+ "must_drop_schema_annotations": True,+ },+ {+ "id": "schema_readonly_property",+ "description": "property named 'readOnly' must survive",+ "payload": {+ "tools": [+ {+ "type": "function",+ "name": "update_field",+ "description": "Update a field in a record.",+ "parameters": {+ "title": "UpdateFieldParameters",+ "type": "object",+ "properties": {+ "field_name": {"type": "string"},+ "value": {"type": "string"},+ "readOnly": {+ "type": "boolean",+ "description": "Whether the field is read-only.",+ },+ },+ "required": ["field_name", "value", "readOnly"],+ "additionalProperties": False,+ },+ }+ ]+ },+ "must_preserve": ["readOnly"],+ "must_drop_schema_annotations": True,+ },+ {+ "id": "schema_multiple_collisions",+ "description": "multiple DROP_KEY collisions in one schema",+ "payload": {+ "tools": [+ {+ "type": "function",+ "name": "create_field",+ "description": "Create a schema field descriptor.",+ "parameters": {+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "title": "CreateFieldParameters",+ "type": "object",+ "properties": {+ "title": {"type": "string"},+ "deprecated": {"type": "boolean"},+ "examples": {+ "type": "array",+ "items": {"type": "string"},+ },+ "readOnly": {"type": "boolean"},+ },+ "required": ["title", "deprecated", "examples", "readOnly"],+ },+ }+ ]+ },+ "must_preserve": ["title", "deprecated", "examples", "readOnly"],+ "must_drop_schema_annotations": True,+ },+ ]++ def evaluate_tool_schema_compaction(+ self,+ cases: list[dict[str, Any]] | None = None,+ ) -> CompressionOnlyResult:+ """Verify tool schema compaction preserves property names that collide with DROP_KEYS.++ The compaction pass must never strip a key that appears as a property+ *name* under a JSON Schema `properties` object, even if the same key+ is in the annotation drop-list (title, deprecated, readOnly, examples, …).++ Assertions per case:+ - token count is smaller after compaction (annotations were stripped)+ - every property name listed in `must_preserve` is present in the+ compacted schema's `properties` dict+ - every `required` array is a subset of the surviving `properties` keys+ (no dangling required entry pointing at a stripped property)+ - schema-level annotations ($schema, title at root level) ARE dropped+ """+ from headroom.proxy.handlers.openai import _compact_openai_responses_tools++ if cases is None:+ cases = self.generate_tool_schema_cases()++ start_time = time.time()+ passed = 0+ failed = 0+ total_original = 0+ total_compressed = 0+ details: list[dict[str, Any]] = []+ errors: list[str] = []++ for case in cases:+ case_id = case["id"]+ payload = case["payload"]+ must_preserve: list[str] = case.get("must_preserve", [])+ must_drop_schema_annotations: bool = case.get("must_drop_schema_annotations", False)++ original_bytes = len(json.dumps(payload).encode())+ total_original += original_bytes++ try:+ compacted, modified, before_bytes, after_bytes = _compact_openai_responses_tools(+ payload+ )+ total_compressed += after_bytes if modified else original_bytes++ case_errors: list[str] = []++ if not modified:+ case_errors.append(+ "compaction reported no modification (annotations not stripped)"+ )++ for tool in compacted.get("tools", []):+ params = tool.get("parameters", {})+ _check_properties_recursive(params, must_preserve, tool["name"], case_errors)++ if must_drop_schema_annotations:+ for tool in compacted.get("tools", []):+ params = tool.get("parameters", {})+ for ann_key in ("$schema", "title"):+ if ann_key in params:+ case_errors.append(+ f"tool '{tool['name']}': schema annotation '{ann_key}' "+ f"was not stripped from parameters root"+ )++ is_pass = len(case_errors) == 0+ if is_pass:+ passed += 1+ else:+ failed += 1+ errors.extend(f"[{case_id}] {e}" for e in case_errors)++ details.append(+ {+ "id": case_id,+ "passed": is_pass,+ "original_bytes": before_bytes,+ "compacted_bytes": after_bytes,+ "compression_ratio": 1 - (after_bytes / before_bytes)+ if before_bytes > 0+ else 0,+ "errors": case_errors,+ }+ )++ except Exception as exc:+ failed += 1+ total_compressed += original_bytes+ errors.append(f"[{case_id}] unexpected exception: {exc}")+ details.append({"id": case_id, "passed": False, "error": str(exc)})++ total_cases = passed + failed+ ratios = [d.get("compression_ratio", 0) for d in details if "compression_ratio" in d]++ return CompressionOnlyResult(+ benchmark="tool_schema_compaction",+ total_cases=total_cases,+ passed_cases=passed,+ failed_cases=failed,+ accuracy_rate=passed / total_cases if total_cases > 0 else 0.0,+ avg_compression_ratio=sum(ratios) / len(ratios) if ratios else 0.0,+ total_original_tokens=total_original // 4,+ total_compressed_tokens=total_compressed // 4,+ total_tokens_saved=(total_original - total_compressed) // 4,+ duration_seconds=time.time() - start_time,+ details=details,+ errors=errors,+ )+++def _check_properties_recursive(+ schema: Any,+ must_preserve: list[str],+ tool_name: str,+ errors: list[str],+) -> None:+ """Walk a JSON Schema object and assert that must_preserve keys survive inside `properties`."""+ if not isinstance(schema, dict):+ return++ properties = schema.get("properties")+ if isinstance(properties, dict):+ required = schema.get("required", [])+ for key in must_preserve:+ if key in required and key not in properties:+ errors.append(+ f"tool '{tool_name}': property '{key}' is in `required` but was "+ f"stripped from `properties` by compaction"+ )+ for sub_schema in properties.values():+ _check_properties_recursive(sub_schema, must_preserve, tool_name, errors)++ items = schema.get("items")+ if isinstance(items, dict):+ _check_properties_recursive(items, must_preserve, tool_name, errors)diff --git a/tests/test_evals_metrics.py b/tests/test_evals_metrics.pyindex 371e36951..9d5432be0 100644--- a/tests/test_evals_metrics.py+++ b/tests/test_evals_metrics.py@@ -130,3 +130,24 @@ def test_information_recall_reports_preserved_and_missing_facts() -> None:empty_original = metrics.compute_information_recall("No facts here", "Still none", ["Alice"])assert empty_original["facts_in_original"] == 0assert empty_original["recall"] == 1.0+++def test_tool_schema_compaction_integrity() -> None:+ """Property names that collide with DROP_KEYS must survive schema compaction.++ Runs the full CompressionOnlyRunner.evaluate_tool_schema_compaction() path+ against the built-in cases and asserts zero failures. This is zero-cost+ (no API calls) and safe for CI smoke runs.+ """+ from headroom.evals.runners.compression_only import CompressionOnlyRunner++ runner = CompressionOnlyRunner()+ result = runner.evaluate_tool_schema_compaction()++ assert result.passed, (+ f"Tool schema compaction integrity failures "+ f"({result.failed_cases}/{result.total_cases}):\n" + "\n".join(result.errors)+ )+ assert result.total_cases == 4, f"Expected 4 built-in cases, got {result.total_cases}"+ # Annotations were stripped so byte count must have shrunk.+ assert result.total_tokens_saved > 0, "Expected at least some annotation tokens to be stripped"