Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
ci(parity): make the parity harness a real per-PR gate (#2567)
Tejas Choprac15e557da1b1
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.ymlindex 80dbd03e9..b50c0bbda 100644--- a/.github/workflows/rust.yml+++ b/.github/workflows/rust.yml@@ -21,7 +21,9 @@ on:- 'Makefile'- '.github/workflows/rust.yml'schedule:- # Nightly parity run at 07:17 UTC (weekdays only). Phase 0 allows failure.+ # Nightly parity run at 07:17 UTC (weekdays only). Redundant with the+ # per-PR gate below, but catches drift from toolchain/dependency updates+ # that land without touching any filtered path.- cron: '17 7 * * 1-5'concurrency:@@ -154,28 +156,27 @@ jobs:continue-on-error: truerun: cargo deny check licenses- parity-nightly:- name: parity (nightly, allowed to fail during Phase 0)- if: github.event_name == 'schedule'+ # Blocking on every PR that touches Rust. Safe to harden now because the+ # harness fails only on a Diff — `parity-run` sets `any_diffs` inside the+ # diffed loop alone, so the 65 fixtures still served by `stub_comparator!`+ # report as Skipped and cannot turn this red. Measured on main today:+ # 111 matched / 65 skipped / 0 diffed.+ #+ # What it protects: the recorded fixtures are frozen Python output, so this+ # gate catches the Rust side drifting away from that snapshot — exactly the+ # failure mode the ongoing port produces. It cannot detect the Python side+ # drifting away from the fixtures; that needs re-recording, not this job.+ parity:+ name: parityruns-on: ubuntu-latest- continue-on-error: truesteps:- uses: actions/checkout@v7- uses: dtolnay/rust-toolchain@stablewith:toolchain: 1.95.0- - uses: actions/setup-python@v6- with:- python-version: '3.11'- uses: Swatinem/rust-cache@v2- - name: Install deps- run: |- python -m venv .venv- source .venv/bin/activate- pip install --upgrade pip- pip install maturin- pip install -e .+ # No Python toolchain: headroom-parity links headroom-core directly and+ # never crosses into the interpreter, so the venv + maturin + `pip+ # install -e .` setup this job used to do was pure overhead.- name: Run parity harness- run: |- source .venv/bin/activate- make test-parity+ run: make test-paritydiff --git a/Makefile b/Makefileindex 5608affb7..3846adc1a 100644--- a/Makefile+++ b/Makefile@@ -12,7 +12,7 @@ FIXTURES ?= tests/parity/fixtureshelp:@echo "Headroom Rust targets:"@echo " make test - cargo test --workspace"- @echo " make test-parity - maturin develop + parity-run against fixtures"+ @echo " make test-parity - parity-run against recorded fixtures"@echo " make bench - cargo bench --workspace"@echo " make build-proxy - release build + strip headroom-proxy, print size"@echo " make build-wheel - release wheel for headroom-py"@@ -36,12 +36,12 @@ help:test:$(CARGO) test --workspace+# headroom-parity has no pyo3 dependency — its comparators call headroom-core+# directly, so this target needs neither a venv nor a built extension module.+# (See crates/headroom-parity/Cargo.toml: "Phase 0 does not invoke Python from+# Rust.") Dropping the `maturin develop` step keeps the harness runnable from a+# bare checkout and takes the Python toolchain off the CI parity job.test-parity:- @if [ -z "$$VIRTUAL_ENV" ]; then \- echo "error: activate a venv first (e.g. source .venv/bin/activate)"; \- exit 1; \- fi- $(MATURIN) develop -m crates/headroom-py/Cargo.toml$(CARGO) run -p headroom-parity -- run --fixtures $(FIXTURES)bench:diff --git a/crates/headroom-parity/src/lib.rs b/crates/headroom-parity/src/lib.rsindex 37310e6ff..2f776b3a1 100644--- a/crates/headroom-parity/src/lib.rs+++ b/crates/headroom-parity/src/lib.rs@@ -466,6 +466,51 @@ impl TransformComparator for ContentDetectorComparator {}}+/// Real comparator for the `text_crusher` transform. The fixture `input` is an+/// object rather than a bare string, mirroring the recorder's three arguments+/// (`tests/parity/record_text_crusher.py`), and `config` is always `null` — the+/// recorder drove the Python default config, so the Rust side uses+/// `TextCrusherConfig::default()` to match.+pub struct TextCrusherComparator;++impl TransformComparator for TextCrusherComparator {+ fn name(&self) -> &str {+ "text_crusher"+ }++ fn run(+ &self,+ input: &serde_json::Value,+ _config: &serde_json::Value,+ ) -> Result<serde_json::Value> {+ use headroom_core::transforms::{TextCrusher, TextCrusherConfig};++ let content = input+ .get("content")+ .and_then(|v| v.as_str())+ .context("text_crusher fixture input needs a string `content`")?;+ let context = input+ .get("context")+ .and_then(|v| v.as_str())+ .unwrap_or_default();+ // Null in the `short_passthrough` fixture, where the recorder passed no+ // ratio at all — `Option<f64>` carries that through to the same default+ // the Python call used.+ let target_ratio = input.get("target_ratio").and_then(|v| v.as_f64());++ let result =+ TextCrusher::new(TextCrusherConfig::default()).compress(content, context, target_ratio);+ Ok(serde_json::json!({+ "compressed": result.compressed,+ "original_tokens": result.original_tokens,+ "compressed_tokens": result.compressed_tokens,+ "compression_ratio": result.compression_ratio,+ "kept_segments": result.kept_segments,+ "total_segments": result.total_segments,+ }))+ }+}+/// Every built-in comparator, in a stable order.pub fn builtin_comparators() -> Vec<Box<dyn TransformComparator>> {vec![@@ -476,6 +521,7 @@ pub fn builtin_comparators() -> Vec<Box<dyn TransformComparator>> {Box::new(CcrComparator),Box::new(SmartCrusherComparator),Box::new(ContentDetectorComparator),+ Box::new(TextCrusherComparator),]}