Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(transforms): first-class C# support in CodeAwareCompressor (#1926)
Yevhen Kovalec55ddcfb3c0
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 22dce9efb..e6958e3a5 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Features+* **transforms:** first-class C# support in `CodeAwareCompressor` via the tree-sitter `csharp` grammar already shipped in the pinned `tree-sitter-language-pack` — no new dependencies ([#1664](https://github.com/headroomlabs-ai/headroom/issues/1664)). Parity with Java/C++/Rust: signatures preserved verbatim, method/constructor/destructor/operator/local-function bodies compressed; block-scoped and file-scoped namespaces, records, structs, interfaces, and enums handled; C#-distinctive auto-detection. Preprocessor conditionals (`#if`…`#endif`) are preserved verbatim as opaque regions (blocks wrapping only `using` directives stay with the imports), `#region` markers no longer swallow the following line during class-member extraction, and top-of-file license banners / `#region License` headers stay on top instead of being relocated below the code. Real-repo runs: 16.1% tokens saved on Newtonsoft.Json (945 files), 37.8% on Polly (797 files), output syntax-valid for 1742/1742 files.* **proxy:** add provider-only HTTP proxy routing via `--http-proxy` and `HEADROOM_HTTP_PROXY`. Upstream LLM provider calls can now use an HTTP proxy without setting process-wide `HTTP_PROXY`/`HTTPS_PROXY` variables that are inherited by tool executions; proxied provider clients use HTTP/1.1 so HTTPS provider APIs can tunnel through CONNECT.* **proxy:** add output shaping for OpenAI Responses traffic on `/v1/responses` HTTP requests and Codex WebSocket `response.create` frames, with stable output-savings holdout keys and counted WS token strata for the experiment.* **observability:** the `headroom.compression.pipeline` span now also carries the OpenTelemetry GenAI semantic-convention attribute `gen_ai.request.model` alongside the existing `headroom.*` attributes, so Headroom's traces group and filter by the standard `gen_ai.*` schema in any OTel-native backend (Grafana, Datadog, etc.). Purely additive; no existing attribute changed. `gen_ai.operation.name`, `gen_ai.provider.name`, and `gen_ai.usage.*` are deliberately deferred (they need per-caller operation threading, reliable upstream-provider resolution, and response-path usage respectively).diff --git a/headroom/transforms/code_compressor.py b/headroom/transforms/code_compressor.pyindex 8ac0efb0a..e9872d936 100644--- a/headroom/transforms/code_compressor.py+++ b/headroom/transforms/code_compressor.py@@ -161,7 +161,7 @@ def _get_parser(language: str) -> Any:except Exception as e:raise ValueError(f"Language '{language}' is not supported by tree-sitter. "- f"Supported: python, javascript, typescript, go, rust, java, c, cpp, perl. "+ f"Supported: python, javascript, typescript, go, rust, java, c, cpp, csharp, perl. "f"Error: {e}") from e@@ -216,6 +216,7 @@ class CodeLanguage(Enum):C = "c"CPP = "cpp"PERL = "perl"+ CSHARP = "csharp"UNKNOWN = "unknown"@@ -302,6 +303,15 @@ class LangConfig:detection_hints: tuple[str, ...] = ()# Optional override for node types that contain class/impl members.class_body_node_types: frozenset[str] | None = None+ # Optional container nodes (e.g. C# block-scoped namespaces) that wrap+ # type declarations in a member list. Routed through class compression so+ # their members compress and the wrapper isn't re-emitted verbatim.+ container_node_types: frozenset[str] | None = None+ # Optional opaque nodes (e.g. C# `#if`/`#endif` conditionals) preserved+ # verbatim without recursing into them. Recursing would capture their+ # descendants individually while the top-level pass re-emits the whole+ # wrapper verbatim, duplicating content. Prefer the false negative.+ opaque_node_types: frozenset[str] | None = None_LANG_CONFIGS: dict[CodeLanguage, LangConfig] = {@@ -416,6 +426,35 @@ _LANG_CONFIGS: dict[CodeLanguage, LangConfig] = {package_node="package_statement",detection_hints=("sub ", "my ", "our ", "use ", "package "),),+ CodeLanguage.CSHARP: LangConfig(+ import_nodes=frozenset({"using_directive", "file_scoped_namespace_declaration"}),+ function_nodes=frozenset(+ {+ "method_declaration",+ "constructor_declaration",+ "destructor_declaration",+ "operator_declaration",+ "local_function_statement",+ }+ ),+ class_nodes=frozenset(+ {+ "class_declaration",+ "struct_declaration",+ "record_declaration",+ "interface_declaration",+ }+ ),+ type_nodes=frozenset({"enum_declaration", "delegate_declaration"}),+ body_node_types=frozenset({"block"}),+ decorator_node=None,+ comment_prefix="//",+ uses_colon_after_signature=False,+ detection_hints=("using ", "namespace ", "public ", "private ", "void "),+ class_body_node_types=frozenset({"declaration_list"}),+ container_node_types=frozenset({"namespace_declaration"}),+ opaque_node_types=frozenset({"preproc_if"}),+ ),}@@ -423,6 +462,7 @@ _LANG_CONFIGS: dict[CodeLanguage, LangConfig] = {class CodeStructure:"""Extracted structure from parsed code."""+ header_code: list[str] = field(default_factory=list)imports: list[str] = field(default_factory=list)type_definitions: list[str] = field(default_factory=list)class_definitions: list[str] = field(default_factory=list)@@ -610,6 +650,16 @@ _LANGUAGE_PREFILTER: dict[CodeLanguage, list[re.Pattern[str]]] = {re.compile(r"^\s*(my|our|local)\s+[\$@%]", re.MULTILINE),re.compile(r"[\$@%]\w+", re.MULTILINE),],+ CodeLanguage.CSHARP: [+ re.compile(r"^\s*using\s+[\w.]+\s*;", re.MULTILINE),+ re.compile(r"^\s*namespace\s+[\w.]+", re.MULTILINE),+ re.compile(+ r"^\s*(public|private|protected|internal|sealed|static|abstract|partial)\s+"+ r"(class|struct|record|interface|enum)\b",+ re.MULTILINE,+ ),+ re.compile(r"\bget;\s*set;", re.MULTILINE),+ ],}@@ -1366,6 +1416,14 @@ class CodeAwareCompressor(Transform):captured_byte_ranges.append((node.start_byte, node.end_byte))return+ if lang_config.container_node_types and node_type in lang_config.container_node_types:+ compressed = self._compress_class_ast(+ node, code, language, lang_config, body_limits, analysis+ )+ structure.class_definitions.append(compressed)+ captured_byte_ranges.append((node.start_byte, node.end_byte))+ return+# Class definitions — compress each method individuallyif node_type in lang_config.class_nodes:leading = _get_leading_comment_text(node, code, captured_byte_ranges)@@ -1388,6 +1446,31 @@ class CodeAwareCompressor(Transform):captured_byte_ranges.append((node.start_byte, node.end_byte))return+ # Opaque regions (e.g. C# preprocessor conditionals) — preserved+ # verbatim, never recursed into. Recursing captures descendants+ # individually while the top-level pass re-emits the uncaptured+ # wrapper verbatim, duplicating its whole content in the output.+ # Blocks wrapping only import directives (`#if NET6_0\nusing X;`)+ # are emitted with the imports: usings must precede type+ # declarations, so appending them as trailing top-level code would+ # produce invalid output (and fall back to no compression).+ if lang_config.opaque_node_types and node_type in lang_config.opaque_node_types:+ child_types = [child.type for child in node.named_children]+ has_import = any(t in lang_config.import_nodes for t in child_types)+ has_declaration = any(+ t in lang_config.class_nodes+ or t in lang_config.type_nodes+ or t in lang_config.function_nodes+ or (lang_config.container_node_types and t in lang_config.container_node_types)+ for t in child_types+ )+ if has_import and not has_declaration:+ structure.imports.append(_get_node_text(node, code))+ else:+ structure.top_level_code.append(_get_node_text(node, code))+ captured_byte_ranges.append((node.start_byte, node.end_byte))+ return+# Recurse into childrenfor child in node.children:visit(child)@@ -1396,13 +1479,22 @@ class CodeAwareCompressor(Transform):# Capture top-level code that wasn't handled by any of the above.# This preserves global variables, constants, if __name__ blocks,- # module-level assignments, etc.+ # module-level assignments, etc. Uncaptured nodes that precede the+ # first captured node are file headers (license banners, C# `#region+ # License` blocks, module comments): they are emitted first, in+ # original order, rather than relocated to the end of the output —+ # e.g. tree-sitter-c-sharp rejects top-level `#region` after a type+ # declaration, which would fail validation and forfeit compression.+ first_captured = min((r[0] for r in captured_byte_ranges), default=None)for child in root.children:child_range = (child.start_byte, child.end_byte)if child_range not in captured_byte_ranges:text = _get_node_text(child, code).strip()if text:- structure.top_level_code.append(text)+ if first_captured is not None and child.end_byte <= first_captured:+ structure.header_code.append(text)+ else:+ structure.top_level_code.append(text)return structure@@ -1717,7 +1809,13 @@ class CodeAwareCompressor(Transform):node_start_line = node.start_point[0]body_start_line = body_node.start_point[0]sig_end = body_start_line - node_start_line- header_lines = node_lines[:sig_end] if sig_end > 0 else [node_lines[0]]+ if sig_end > 0:+ header_lines = node_lines[:sig_end]+ brace_line = node_lines[sig_end]+ if brace_line.strip().startswith("{"):+ header_lines = [*header_lines, brace_line]+ else:+ header_lines = [node_lines[0]]# Process each child of the class body individuallybody_parts: list[str] = []@@ -1730,6 +1828,8 @@ class CodeAwareCompressor(Transform):# Use line-based extraction for children toochild_start = child.start_point[0]child_end = child.end_point[0]+ if child.end_point[1] == 0 and child_end > child_start:+ child_end -= 1child_text = "\n".join(code_lines[child_start : child_end + 1])# Methods/functions inside the class — compress individually@@ -1759,8 +1859,10 @@ class CodeAwareCompressor(Transform):else:body_parts.append(child_text)processed_ranges.append((child.start_byte, child.end_byte))- # Nested classes — recurse- elif child.type in lang_config.class_nodes:+ # Nested classes / containers (e.g. nested namespaces) — recurse+ elif child.type in lang_config.class_nodes or (+ lang_config.container_node_types and child.type in lang_config.container_node_types+ ):compressed = self._compress_class_ast(child, code, language, lang_config, body_limits, analysis)@@ -1816,6 +1918,11 @@ class CodeAwareCompressor(Transform):"""Assemble compressed code from structure."""parts: list[str] = []+ # File header (license banners, top-of-file comments) stays on top+ if structure.header_code:+ parts.extend(structure.header_code)+ parts.append("")+# Imports firstif structure.imports:parts.extend(structure.imports)diff --git a/headroom/transforms/content_detector.py b/headroom/transforms/content_detector.pyindex 3a22020cb..850413d7e 100644--- a/headroom/transforms/content_detector.py+++ b/headroom/transforms/content_detector.py@@ -100,6 +100,15 @@ _CODE_PATTERNS = {re.compile(r"^\s*@\w+"), # annotationsre.compile(r"^\s*package\s+[\w.]+;"),],+ "csharp": [+ re.compile(r"^\s*using\s+[\w.]+\s*;"), # using directive (not C++ `using namespace x;`)+ re.compile(r"^\s*namespace\s+[\w.]+"),+ re.compile(+ r"^\s*(public|private|protected|internal|sealed|static|abstract|partial)\s+"+ r"(class|struct|record|interface|enum)\b"+ ),+ re.compile(r"^.*\b(get|set|init);"), # auto-property accessors+ ],}# Log/build output patternsdiff --git a/tests/test_transforms/test_code_compressor.py b/tests/test_transforms/test_code_compressor.pyindex 059e35cac..574b438a1 100644--- a/tests/test_transforms/test_code_compressor.py+++ b/tests/test_transforms/test_code_compressor.py@@ -869,6 +869,30 @@ class TestTreeSitterIntegration:" e\n","}\n}",),+ (+ "csharp",+ (+ "namespace Acme\n"+ "{\n"+ " public class Calc\n"+ " {\n"+ " public int Compute(int x)\n"+ " {\n"+ " int a = x + 1;\n"+ " int b = a * 2;\n"+ " int c = b - 3;\n"+ " int d = c / 4;\n"+ " int e = d + 5;\n"+ " return e;\n"+ " }\n"+ " }\n"+ "}\n"+ ),+ "public int Compute(int x)",+ 5,+ "return e;",+ "}\n}",+ ),],)def test_compresses_methods_inside_class_member_containers(@@ -1648,3 +1672,299 @@ class TestRealASTRuns:assert result.syntax_valid is True# Signature is preserved verbatim.assert "pub fn add(a: i64, b: i64) -> i64" in result.compressed++[email protected](not TREE_SITTER_INSTALLED, reason="tree-sitter grammar pack not installed")+class TestCSharpSupport:+ """C# (``csharp`` grammar) parity with Java/C++/Rust: signatures preserved+ verbatim, method/constructor bodies compressed, block-scoped namespaces+ routed through container compression (no verbatim re-dump), file-scoped+ namespace headers kept ahead of types, malformed input passed through.+ """++ def _compressor(self):+ return CodeAwareCompressor(+ CodeCompressorConfig(+ min_tokens_for_compression=1,+ max_body_lines=1,+ enable_ccr=False,+ )+ )++ def test_block_scoped_namespace_compresses_without_redumping(self):+ """Block-scoped ``namespace { }`` wraps types in a declaration_list; the+ container must compress its members and NOT be re-emitted verbatim+ (which would duplicate the class and defeat compression — the C#+ analogue of the #1318 class-member trap, one nesting level deeper)."""+ code = (+ "using System;\n"+ "\n"+ "namespace Acme.Widgets\n"+ "{\n"+ " public class WidgetService\n"+ " {\n"+ " public int Process(int x)\n"+ " {\n"+ " var a = x + 1;\n"+ " var b = a + 2;\n"+ " var c = b + 3;\n"+ " Console.WriteLine(c);\n"+ " return c;\n"+ " }\n"+ " }\n"+ "}\n"+ )+ result = self._compressor().compress(code, language="csharp")++ assert result.language == CodeLanguage.CSHARP+ assert result.syntax_valid is True+ assert result.compression_ratio < 1.0+ # namespace + type headers preserved verbatim+ assert "namespace Acme.Widgets" in result.compressed+ assert "public class WidgetService" in result.compressed+ assert "public int Process(int x)" in result.compressed+ # method body actually compressed+ assert "lines omitted" in result.compressed+ assert "return c;" not in result.compressed+ # the class is emitted exactly once (no re-dump / duplication)+ assert result.compressed.count("class WidgetService") == 1+ # block-namespace wrapper still closes+ assert result.compressed.rstrip().endswith("}")++ def test_file_scoped_namespace_directive_precedes_types(self):+ """A file-scoped ``namespace X;`` must stay ahead of the type+ declarations; emitting it after a type would not be valid C#."""+ code = (+ "using System;\n"+ "\n"+ "namespace Acme.Tools;\n"+ "\n"+ "public class Helper\n"+ "{\n"+ " public int Add(int a, int b)\n"+ " {\n"+ " var s = a + b;\n"+ " var t = s + 1;\n"+ " var u = t + 2;\n"+ " return u;\n"+ " }\n"+ "}\n"+ )+ result = self._compressor().compress(code, language="csharp")++ assert result.language == CodeLanguage.CSHARP+ assert result.syntax_valid is True+ assert result.compression_ratio < 1.0+ assert "namespace Acme.Tools;" in result.compressed+ assert "public class Helper" in result.compressed+ assert result.compressed.index("namespace Acme.Tools;") < result.compressed.index(+ "class Helper"+ )++ def test_record_preserves_positional_parameters(self):+ """Record primary-constructor parameters are part of the signature and+ must be preserved verbatim while the method body compresses."""+ code = (+ "namespace Acme.Models;\n"+ "\n"+ "public record Point(int X, int Y)\n"+ "{\n"+ " public double Dist()\n"+ " {\n"+ " var sq = X * X + Y * Y;\n"+ " var r = System.Math.Sqrt(sq);\n"+ " System.Console.WriteLine(r);\n"+ " return r;\n"+ " }\n"+ "}\n"+ )+ result = self._compressor().compress(code, language="csharp")++ assert result.language == CodeLanguage.CSHARP+ assert result.syntax_valid is True+ assert result.compression_ratio < 1.0+ assert "public record Point(int X, int Y)" in result.compressed+ assert "public double Dist()" in result.compressed+ assert "lines omitted" in result.compressed++ def test_preprocessor_wrapped_file_is_not_duplicated(self):+ """A file wrapped in ``#if``/``#endif`` (idiomatic in multi-targeted+ .NET code) must not have its content emitted twice. The visitor+ recurses into ``preproc_if`` and captures the declarations, but the+ top-level pass used to re-emit the uncaptured wrapper verbatim,+ duplicating the whole file (observed on real repos: output up to+ ~1.9x the input). Opaque handling keeps it verbatim exactly once."""+ code = (+ "#if !HAVE_TRACE_WRITER\n"+ "using System;\n"+ "\n"+ "namespace Acme\n"+ "{\n"+ " public enum TraceLevel\n"+ " {\n"+ " Off = 0,\n"+ " Error = 1,\n"+ " }\n"+ "}\n"+ "#endif\n"+ )+ result = self._compressor().compress(code, language="csharp")++ assert result.syntax_valid is True+ # Content appears exactly once — no wrapper re-dump.+ assert result.compressed.count("enum TraceLevel") == 1+ assert result.compressed.count("using System;") == 1+ # Never larger than the input (verbatim pass-through is acceptable).+ assert result.compression_ratio <= 1.0+ # Conditional directives stay balanced.+ assert result.compressed.count("#if") == result.compressed.count("#endif")++ def test_preprocessor_wrapped_usings_stay_ahead_of_types(self):+ """``#if``-wrapped using directives (idiomatic for multi-targeting)+ must be emitted with the imports. Appending them as trailing top-level+ code puts usings after type declarations — invalid C#, which trips the+ output syntax check and falls back to no compression at all (observed+ on real repos: hundreds of files silently uncompressed)."""+ code = (+ "using System;\n"+ "#if HAVE_BIG_INTEGER\n"+ "using System.Numerics;\n"+ "#endif\n"+ "\n"+ "namespace Acme\n"+ "{\n"+ " public class Calc\n"+ " {\n"+ " public int Compute(int x)\n"+ " {\n"+ " var a = x + 1;\n"+ " var b = a * 2;\n"+ " var c = b - 3;\n"+ " var d = c / 4;\n"+ " return d;\n"+ " }\n"+ " }\n"+ "}\n"+ )+ result = self._compressor().compress(code, language="csharp")++ assert result.syntax_valid is True+ # Compression must actually happen (no silent fallback).+ assert result.compression_ratio < 1.0+ assert "lines omitted" in result.compressed+ # The conditional import block survives verbatim, ahead of the types.+ assert "#if HAVE_BIG_INTEGER" in result.compressed+ assert "using System.Numerics;" in result.compressed+ assert result.compressed.index("#endif") < result.compressed.index("class Calc")++ def test_region_markers_inside_class_compress_cleanly(self):+ """``#region``/``#endregion`` markers span their trailing newline, so+ their end_point sits at column 0 of the NEXT line. Line-based child+ extraction used to swallow that line — duplicating the following+ method's signature and the class's closing brace (invalid output →+ silent fallback to no compression on real repos)."""+ code = (+ "namespace Acme\n"+ "{\n"+ " public class Svc\n"+ " {\n"+ " #region Api\n"+ " public int F(int x)\n"+ " {\n"+ " var a = x + 1;\n"+ " var b = a * 2;\n"+ " var c = b - 3;\n"+ " return c;\n"+ " }\n"+ " #endregion\n"+ " }\n"+ "}\n"+ )+ result = self._compressor().compress(code, language="csharp")++ assert result.syntax_valid is True+ assert result.compression_ratio < 1.0+ assert "lines omitted" in result.compressed+ # Signature emitted exactly once (regions used to duplicate it).+ assert result.compressed.count("public int F(int x)") == 1+ # Region markers survive, braces stay balanced.+ assert "#region Api" in result.compressed+ assert "#endregion" in result.compressed+ assert result.compressed.count("{") == result.compressed.count("}")++ def test_license_region_header_stays_on_top(self):+ """A ``#region License`` banner above the usings (near-universal in+ real .NET code) must stay at the top of the file. Relocating it after+ the type declarations — the old behavior for uncaptured top-level+ nodes — is rejected by tree-sitter-c-sharp, failing output validation+ and silently forfeiting compression for the whole file."""+ code = (+ "#region License\n"+ "// Copyright (c) 2007 Example Corp.\n"+ "// Licensed under the MIT license.\n"+ "#endregion\n"+ "\n"+ "using System;\n"+ "\n"+ "namespace Acme\n"+ "{\n"+ " public class Svc\n"+ " {\n"+ " public int F(int x)\n"+ " {\n"+ " var a = x + 1;\n"+ " var b = a * 2;\n"+ " var c = b - 3;\n"+ " return c;\n"+ " }\n"+ " }\n"+ "}\n"+ )+ result = self._compressor().compress(code, language="csharp")++ assert result.syntax_valid is True+ assert result.compression_ratio < 1.0+ assert "lines omitted" in result.compressed+ # Header block survives, in order, ahead of everything else.+ assert result.compressed.index("#region License") < result.compressed.index("using System;")+ assert result.compressed.index("// Copyright") < result.compressed.index("#endregion")+ assert result.compressed.index("#endregion") < result.compressed.index("class Svc")++ def test_malformed_csharp_passes_through_unchanged(self):+ """Malformed C# must pass through unchanged (no data loss; prefer false+ negatives over serving broken or altered code)."""+ code = (+ "namespace Broken\n"+ "{\n"+ " public class Oops\n"+ " {\n"+ " public int F(int x)\n"+ " {\n"+ " var a = x + 1\n" # missing ';' and unclosed braces+ " return a\n"+ )+ result = self._compressor().compress(code, language="csharp")++ assert result.compressed == code+ assert result.compression_ratio == 1.0++ def test_detect_language_identifies_csharp(self):+ """Auto-detection (no explicit language hint) recognizes C#."""+ code = (+ "using System;\n"+ "\n"+ "namespace Acme\n"+ "{\n"+ " public class Svc\n"+ " {\n"+ " public int Total { get; set; }\n"+ " public int Add(int a, int b)\n"+ " {\n"+ " return a + b;\n"+ " }\n"+ " }\n"+ "}\n"+ )+ lang, confidence = detect_language(code)+ assert lang == CodeLanguage.CSHARP+ assert confidence > 0.0