Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(evals): add zero-cost tool schema compaction integrity eval (#817)

Ashish53a08c63bf56
diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml
index 20e24a6e4..bbaef6d4c 100644
--- a/.github/workflows/eval.yml
+++ b/.github/workflows/eval.yml
@@ -54,6 +54,17 @@ jobs:
print(f'CCR Round-trip: {result.passed_cases}/{result.total_cases} passed')
assert result.passed, f'CCR failures: {result.errors}'
"
+
+ - name: Run tool schema compaction integrity eval (zero cost)
+ run: |
+ python -c "
+ from headroom.evals.runners.compression_only import CompressionOnlyRunner
+ runner = CompressionOnlyRunner()
+ result = runner.evaluate_tool_schema_compaction()
+ print(f'Tool schema compaction: {result.passed_cases}/{result.total_cases} passed, {result.total_tokens_saved} annotation tokens stripped')
+ assert result.passed, f'Schema compaction failures: {result.errors}'
+ "
+
# OPENAI_API_KEY is intentionally not set in the public OSS repo
# (the secret list is empty). The CCR round-trip step above is the
# mandatory gate; this step only runs when an operator has wired
diff --git a/headroom/evals/runners/compression_only.py b/headroom/evals/runners/compression_only.py
index b999ab116..4ef71dbc0 100644
--- a/headroom/evals/runners/compression_only.py
+++ b/headroom/evals/runners/compression_only.py
@@ -5,6 +5,7 @@ Used for:
- CCR lossless round-trip verification
- Information retention (probe facts survive compression)
- Needle retention (specific values preserved in compressed output)
+- Tool schema compaction integrity (property names survive annotation stripping)
"""
from __future__ import annotations
@@ -335,3 +336,275 @@ class CompressionOnlyRunner:
)
return cases[:n]
+
+ def generate_tool_schema_cases(self) -> list[dict[str, Any]]:
+ """Generate tool schema test cases for compaction integrity verification.
+
+ Each case exercises a different way a DROP_KEY can appear as a
+ property *name* inside a JSON Schema `properties` object.
+ The cases also include annotation keys at the schema level so we
+ can verify those ARE still stripped.
+ """
+ return [
+ {
+ "id": "schema_title_property",
+ "description": "property named 'title' must survive; schema-level title must be dropped",
+ "payload": {
+ "tools": [
+ {
+ "type": "function",
+ "name": "eval_cells",
+ "description": "Evaluate notebook cells.",
+ "parameters": {
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "title": "EvalCellsParameters",
+ "type": "object",
+ "properties": {
+ "cells": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "title": "CellItem",
+ "properties": {
+ "language": {"type": "string"},
+ "code": {"type": "string"},
+ "title": {"type": "string"},
+ },
+ "required": ["language", "code", "title"],
+ },
+ }
+ },
+ "required": ["cells"],
+ },
+ }
+ ]
+ },
+ "must_preserve": ["title"],
+ "must_drop_schema_annotations": True,
+ },
+ {
+ "id": "schema_deprecated_property",
+ "description": "property named 'deprecated' must survive",
+ "payload": {
+ "tools": [
+ {
+ "type": "function",
+ "name": "list_apis",
+ "description": "List available APIs with their status.",
+ "parameters": {
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "title": "ListApisParameters",
+ "type": "object",
+ "properties": {
+ "deprecated": {
+ "type": "boolean",
+ "description": "Include deprecated APIs in results.",
+ },
+ "name": {"type": "string"},
+ },
+ "required": ["deprecated"],
+ },
+ }
+ ]
+ },
+ "must_preserve": ["deprecated"],
+ "must_drop_schema_annotations": True,
+ },
+ {
+ "id": "schema_readonly_property",
+ "description": "property named 'readOnly' must survive",
+ "payload": {
+ "tools": [
+ {
+ "type": "function",
+ "name": "update_field",
+ "description": "Update a field in a record.",
+ "parameters": {
+ "title": "UpdateFieldParameters",
+ "type": "object",
+ "properties": {
+ "field_name": {"type": "string"},
+ "value": {"type": "string"},
+ "readOnly": {
+ "type": "boolean",
+ "description": "Whether the field is read-only.",
+ },
+ },
+ "required": ["field_name", "value", "readOnly"],
+ "additionalProperties": False,
+ },
+ }
+ ]
+ },
+ "must_preserve": ["readOnly"],
+ "must_drop_schema_annotations": True,
+ },
+ {
+ "id": "schema_multiple_collisions",
+ "description": "multiple DROP_KEY collisions in one schema",
+ "payload": {
+ "tools": [
+ {
+ "type": "function",
+ "name": "create_field",
+ "description": "Create a schema field descriptor.",
+ "parameters": {
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "title": "CreateFieldParameters",
+ "type": "object",
+ "properties": {
+ "title": {"type": "string"},
+ "deprecated": {"type": "boolean"},
+ "examples": {
+ "type": "array",
+ "items": {"type": "string"},
+ },
+ "readOnly": {"type": "boolean"},
+ },
+ "required": ["title", "deprecated", "examples", "readOnly"],
+ },
+ }
+ ]
+ },
+ "must_preserve": ["title", "deprecated", "examples", "readOnly"],
+ "must_drop_schema_annotations": True,
+ },
+ ]
+
+ def evaluate_tool_schema_compaction(
+ self,
+ cases: list[dict[str, Any]] | None = None,
+ ) -> CompressionOnlyResult:
+ """Verify tool schema compaction preserves property names that collide with DROP_KEYS.
+
+ The compaction pass must never strip a key that appears as a property
+ *name* under a JSON Schema `properties` object, even if the same key
+ is in the annotation drop-list (title, deprecated, readOnly, examples, …).
+
+ Assertions per case:
+ - token count is smaller after compaction (annotations were stripped)
+ - every property name listed in `must_preserve` is present in the
+ compacted schema's `properties` dict
+ - every `required` array is a subset of the surviving `properties` keys
+ (no dangling required entry pointing at a stripped property)
+ - schema-level annotations ($schema, title at root level) ARE dropped
+ """
+ from headroom.proxy.handlers.openai import _compact_openai_responses_tools
+
+ if cases is None:
+ cases = self.generate_tool_schema_cases()
+
+ start_time = time.time()
+ passed = 0
+ failed = 0
+ total_original = 0
+ total_compressed = 0
+ details: list[dict[str, Any]] = []
+ errors: list[str] = []
+
+ for case in cases:
+ case_id = case["id"]
+ payload = case["payload"]
+ must_preserve: list[str] = case.get("must_preserve", [])
+ must_drop_schema_annotations: bool = case.get("must_drop_schema_annotations", False)
+
+ original_bytes = len(json.dumps(payload).encode())
+ total_original += original_bytes
+
+ try:
+ compacted, modified, before_bytes, after_bytes = _compact_openai_responses_tools(
+ payload
+ )
+ total_compressed += after_bytes if modified else original_bytes
+
+ case_errors: list[str] = []
+
+ if not modified:
+ case_errors.append(
+ "compaction reported no modification (annotations not stripped)"
+ )
+
+ for tool in compacted.get("tools", []):
+ params = tool.get("parameters", {})
+ _check_properties_recursive(params, must_preserve, tool["name"], case_errors)
+
+ if must_drop_schema_annotations:
+ for tool in compacted.get("tools", []):
+ params = tool.get("parameters", {})
+ for ann_key in ("$schema", "title"):
+ if ann_key in params:
+ case_errors.append(
+ f"tool '{tool['name']}': schema annotation '{ann_key}' "
+ f"was not stripped from parameters root"
+ )
+
+ is_pass = len(case_errors) == 0
+ if is_pass:
+ passed += 1
+ else:
+ failed += 1
+ errors.extend(f"[{case_id}] {e}" for e in case_errors)
+
+ details.append(
+ {
+ "id": case_id,
+ "passed": is_pass,
+ "original_bytes": before_bytes,
+ "compacted_bytes": after_bytes,
+ "compression_ratio": 1 - (after_bytes / before_bytes)
+ if before_bytes > 0
+ else 0,
+ "errors": case_errors,
+ }
+ )
+
+ except Exception as exc:
+ failed += 1
+ total_compressed += original_bytes
+ errors.append(f"[{case_id}] unexpected exception: {exc}")
+ details.append({"id": case_id, "passed": False, "error": str(exc)})
+
+ total_cases = passed + failed
+ ratios = [d.get("compression_ratio", 0) for d in details if "compression_ratio" in d]
+
+ return CompressionOnlyResult(
+ benchmark="tool_schema_compaction",
+ total_cases=total_cases,
+ passed_cases=passed,
+ failed_cases=failed,
+ accuracy_rate=passed / total_cases if total_cases > 0 else 0.0,
+ avg_compression_ratio=sum(ratios) / len(ratios) if ratios else 0.0,
+ total_original_tokens=total_original // 4,
+ total_compressed_tokens=total_compressed // 4,
+ total_tokens_saved=(total_original - total_compressed) // 4,
+ duration_seconds=time.time() - start_time,
+ details=details,
+ errors=errors,
+ )
+
+
+def _check_properties_recursive(
+ schema: Any,
+ must_preserve: list[str],
+ tool_name: str,
+ errors: list[str],
+) -> None:
+ """Walk a JSON Schema object and assert that must_preserve keys survive inside `properties`."""
+ if not isinstance(schema, dict):
+ return
+
+ properties = schema.get("properties")
+ if isinstance(properties, dict):
+ required = schema.get("required", [])
+ for key in must_preserve:
+ if key in required and key not in properties:
+ errors.append(
+ f"tool '{tool_name}': property '{key}' is in `required` but was "
+ f"stripped from `properties` by compaction"
+ )
+ for sub_schema in properties.values():
+ _check_properties_recursive(sub_schema, must_preserve, tool_name, errors)
+
+ items = schema.get("items")
+ if isinstance(items, dict):
+ _check_properties_recursive(items, must_preserve, tool_name, errors)
diff --git a/tests/test_evals_metrics.py b/tests/test_evals_metrics.py
index 371e36951..9d5432be0 100644
--- a/tests/test_evals_metrics.py
+++ b/tests/test_evals_metrics.py
@@ -130,3 +130,24 @@ def test_information_recall_reports_preserved_and_missing_facts() -> None:
empty_original = metrics.compute_information_recall("No facts here", "Still none", ["Alice"])
assert empty_original["facts_in_original"] == 0
assert empty_original["recall"] == 1.0
+
+
+def test_tool_schema_compaction_integrity() -> None:
+ """Property names that collide with DROP_KEYS must survive schema compaction.
+
+ Runs the full CompressionOnlyRunner.evaluate_tool_schema_compaction() path
+ against the built-in cases and asserts zero failures. This is zero-cost
+ (no API calls) and safe for CI smoke runs.
+ """
+ from headroom.evals.runners.compression_only import CompressionOnlyRunner
+
+ runner = CompressionOnlyRunner()
+ result = runner.evaluate_tool_schema_compaction()
+
+ assert result.passed, (
+ f"Tool schema compaction integrity failures "
+ f"({result.failed_cases}/{result.total_cases}):\n" + "\n".join(result.errors)
+ )
+ assert result.total_cases == 4, f"Expected 4 built-in cases, got {result.total_cases}"
+ # Annotations were stripped so byte count must have shrunk.
+ assert result.total_tokens_saved > 0, "Expected at least some annotation tokens to be stripped"