Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(transforms): first-class C# support in CodeAwareCompressor (#1926)

Yevhen Kovalec55ddcfb3c0
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 22dce9efb..e6958e3a5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Features
+* **transforms:** first-class C# support in `CodeAwareCompressor` via the tree-sitter `csharp` grammar already shipped in the pinned `tree-sitter-language-pack` — no new dependencies ([#1664](https://github.com/headroomlabs-ai/headroom/issues/1664)). Parity with Java/C++/Rust: signatures preserved verbatim, method/constructor/destructor/operator/local-function bodies compressed; block-scoped and file-scoped namespaces, records, structs, interfaces, and enums handled; C#-distinctive auto-detection. Preprocessor conditionals (`#if`…`#endif`) are preserved verbatim as opaque regions (blocks wrapping only `using` directives stay with the imports), `#region` markers no longer swallow the following line during class-member extraction, and top-of-file license banners / `#region License` headers stay on top instead of being relocated below the code. Real-repo runs: 16.1% tokens saved on Newtonsoft.Json (945 files), 37.8% on Polly (797 files), output syntax-valid for 1742/1742 files.
* **proxy:** add provider-only HTTP proxy routing via `--http-proxy` and `HEADROOM_HTTP_PROXY`. Upstream LLM provider calls can now use an HTTP proxy without setting process-wide `HTTP_PROXY`/`HTTPS_PROXY` variables that are inherited by tool executions; proxied provider clients use HTTP/1.1 so HTTPS provider APIs can tunnel through CONNECT.
* **proxy:** add output shaping for OpenAI Responses traffic on `/v1/responses` HTTP requests and Codex WebSocket `response.create` frames, with stable output-savings holdout keys and counted WS token strata for the experiment.
* **observability:** the `headroom.compression.pipeline` span now also carries the OpenTelemetry GenAI semantic-convention attribute `gen_ai.request.model` alongside the existing `headroom.*` attributes, so Headroom's traces group and filter by the standard `gen_ai.*` schema in any OTel-native backend (Grafana, Datadog, etc.). Purely additive; no existing attribute changed. `gen_ai.operation.name`, `gen_ai.provider.name`, and `gen_ai.usage.*` are deliberately deferred (they need per-caller operation threading, reliable upstream-provider resolution, and response-path usage respectively).
diff --git a/headroom/transforms/code_compressor.py b/headroom/transforms/code_compressor.py
index 8ac0efb0a..e9872d936 100644
--- a/headroom/transforms/code_compressor.py
+++ b/headroom/transforms/code_compressor.py
@@ -161,7 +161,7 @@ def _get_parser(language: str) -> Any:
except Exception as e:
raise ValueError(
f"Language '{language}' is not supported by tree-sitter. "
- f"Supported: python, javascript, typescript, go, rust, java, c, cpp, perl. "
+ f"Supported: python, javascript, typescript, go, rust, java, c, cpp, csharp, perl. "
f"Error: {e}"
) from e
@@ -216,6 +216,7 @@ class CodeLanguage(Enum):
C = "c"
CPP = "cpp"
PERL = "perl"
+ CSHARP = "csharp"
UNKNOWN = "unknown"
@@ -302,6 +303,15 @@ class LangConfig:
detection_hints: tuple[str, ...] = ()
# Optional override for node types that contain class/impl members.
class_body_node_types: frozenset[str] | None = None
+ # Optional container nodes (e.g. C# block-scoped namespaces) that wrap
+ # type declarations in a member list. Routed through class compression so
+ # their members compress and the wrapper isn't re-emitted verbatim.
+ container_node_types: frozenset[str] | None = None
+ # Optional opaque nodes (e.g. C# `#if`/`#endif` conditionals) preserved
+ # verbatim without recursing into them. Recursing would capture their
+ # descendants individually while the top-level pass re-emits the whole
+ # wrapper verbatim, duplicating content. Prefer the false negative.
+ opaque_node_types: frozenset[str] | None = None
_LANG_CONFIGS: dict[CodeLanguage, LangConfig] = {
@@ -416,6 +426,35 @@ _LANG_CONFIGS: dict[CodeLanguage, LangConfig] = {
package_node="package_statement",
detection_hints=("sub ", "my ", "our ", "use ", "package "),
),
+ CodeLanguage.CSHARP: LangConfig(
+ import_nodes=frozenset({"using_directive", "file_scoped_namespace_declaration"}),
+ function_nodes=frozenset(
+ {
+ "method_declaration",
+ "constructor_declaration",
+ "destructor_declaration",
+ "operator_declaration",
+ "local_function_statement",
+ }
+ ),
+ class_nodes=frozenset(
+ {
+ "class_declaration",
+ "struct_declaration",
+ "record_declaration",
+ "interface_declaration",
+ }
+ ),
+ type_nodes=frozenset({"enum_declaration", "delegate_declaration"}),
+ body_node_types=frozenset({"block"}),
+ decorator_node=None,
+ comment_prefix="//",
+ uses_colon_after_signature=False,
+ detection_hints=("using ", "namespace ", "public ", "private ", "void "),
+ class_body_node_types=frozenset({"declaration_list"}),
+ container_node_types=frozenset({"namespace_declaration"}),
+ opaque_node_types=frozenset({"preproc_if"}),
+ ),
}
@@ -423,6 +462,7 @@ _LANG_CONFIGS: dict[CodeLanguage, LangConfig] = {
class CodeStructure:
"""Extracted structure from parsed code."""
+ header_code: list[str] = field(default_factory=list)
imports: list[str] = field(default_factory=list)
type_definitions: list[str] = field(default_factory=list)
class_definitions: list[str] = field(default_factory=list)
@@ -610,6 +650,16 @@ _LANGUAGE_PREFILTER: dict[CodeLanguage, list[re.Pattern[str]]] = {
re.compile(r"^\s*(my|our|local)\s+[\$@%]", re.MULTILINE),
re.compile(r"[\$@%]\w+", re.MULTILINE),
],
+ CodeLanguage.CSHARP: [
+ re.compile(r"^\s*using\s+[\w.]+\s*;", re.MULTILINE),
+ re.compile(r"^\s*namespace\s+[\w.]+", re.MULTILINE),
+ re.compile(
+ r"^\s*(public|private|protected|internal|sealed|static|abstract|partial)\s+"
+ r"(class|struct|record|interface|enum)\b",
+ re.MULTILINE,
+ ),
+ re.compile(r"\bget;\s*set;", re.MULTILINE),
+ ],
}
@@ -1366,6 +1416,14 @@ class CodeAwareCompressor(Transform):
captured_byte_ranges.append((node.start_byte, node.end_byte))
return
+ if lang_config.container_node_types and node_type in lang_config.container_node_types:
+ compressed = self._compress_class_ast(
+ node, code, language, lang_config, body_limits, analysis
+ )
+ structure.class_definitions.append(compressed)
+ captured_byte_ranges.append((node.start_byte, node.end_byte))
+ return
+
# Class definitions — compress each method individually
if node_type in lang_config.class_nodes:
leading = _get_leading_comment_text(node, code, captured_byte_ranges)
@@ -1388,6 +1446,31 @@ class CodeAwareCompressor(Transform):
captured_byte_ranges.append((node.start_byte, node.end_byte))
return
+ # Opaque regions (e.g. C# preprocessor conditionals) — preserved
+ # verbatim, never recursed into. Recursing captures descendants
+ # individually while the top-level pass re-emits the uncaptured
+ # wrapper verbatim, duplicating its whole content in the output.
+ # Blocks wrapping only import directives (`#if NET6_0\nusing X;`)
+ # are emitted with the imports: usings must precede type
+ # declarations, so appending them as trailing top-level code would
+ # produce invalid output (and fall back to no compression).
+ if lang_config.opaque_node_types and node_type in lang_config.opaque_node_types:
+ child_types = [child.type for child in node.named_children]
+ has_import = any(t in lang_config.import_nodes for t in child_types)
+ has_declaration = any(
+ t in lang_config.class_nodes
+ or t in lang_config.type_nodes
+ or t in lang_config.function_nodes
+ or (lang_config.container_node_types and t in lang_config.container_node_types)
+ for t in child_types
+ )
+ if has_import and not has_declaration:
+ structure.imports.append(_get_node_text(node, code))
+ else:
+ structure.top_level_code.append(_get_node_text(node, code))
+ captured_byte_ranges.append((node.start_byte, node.end_byte))
+ return
+
# Recurse into children
for child in node.children:
visit(child)
@@ -1396,13 +1479,22 @@ class CodeAwareCompressor(Transform):
# Capture top-level code that wasn't handled by any of the above.
# This preserves global variables, constants, if __name__ blocks,
- # module-level assignments, etc.
+ # module-level assignments, etc. Uncaptured nodes that precede the
+ # first captured node are file headers (license banners, C# `#region
+ # License` blocks, module comments): they are emitted first, in
+ # original order, rather than relocated to the end of the output —
+ # e.g. tree-sitter-c-sharp rejects top-level `#region` after a type
+ # declaration, which would fail validation and forfeit compression.
+ first_captured = min((r[0] for r in captured_byte_ranges), default=None)
for child in root.children:
child_range = (child.start_byte, child.end_byte)
if child_range not in captured_byte_ranges:
text = _get_node_text(child, code).strip()
if text:
- structure.top_level_code.append(text)
+ if first_captured is not None and child.end_byte <= first_captured:
+ structure.header_code.append(text)
+ else:
+ structure.top_level_code.append(text)
return structure
@@ -1717,7 +1809,13 @@ class CodeAwareCompressor(Transform):
node_start_line = node.start_point[0]
body_start_line = body_node.start_point[0]
sig_end = body_start_line - node_start_line
- header_lines = node_lines[:sig_end] if sig_end > 0 else [node_lines[0]]
+ if sig_end > 0:
+ header_lines = node_lines[:sig_end]
+ brace_line = node_lines[sig_end]
+ if brace_line.strip().startswith("{"):
+ header_lines = [*header_lines, brace_line]
+ else:
+ header_lines = [node_lines[0]]
# Process each child of the class body individually
body_parts: list[str] = []
@@ -1730,6 +1828,8 @@ class CodeAwareCompressor(Transform):
# Use line-based extraction for children too
child_start = child.start_point[0]
child_end = child.end_point[0]
+ if child.end_point[1] == 0 and child_end > child_start:
+ child_end -= 1
child_text = "\n".join(code_lines[child_start : child_end + 1])
# Methods/functions inside the class — compress individually
@@ -1759,8 +1859,10 @@ class CodeAwareCompressor(Transform):
else:
body_parts.append(child_text)
processed_ranges.append((child.start_byte, child.end_byte))
- # Nested classes — recurse
- elif child.type in lang_config.class_nodes:
+ # Nested classes / containers (e.g. nested namespaces) — recurse
+ elif child.type in lang_config.class_nodes or (
+ lang_config.container_node_types and child.type in lang_config.container_node_types
+ ):
compressed = self._compress_class_ast(
child, code, language, lang_config, body_limits, analysis
)
@@ -1816,6 +1918,11 @@ class CodeAwareCompressor(Transform):
"""Assemble compressed code from structure."""
parts: list[str] = []
+ # File header (license banners, top-of-file comments) stays on top
+ if structure.header_code:
+ parts.extend(structure.header_code)
+ parts.append("")
+
# Imports first
if structure.imports:
parts.extend(structure.imports)
diff --git a/headroom/transforms/content_detector.py b/headroom/transforms/content_detector.py
index 3a22020cb..850413d7e 100644
--- a/headroom/transforms/content_detector.py
+++ b/headroom/transforms/content_detector.py
@@ -100,6 +100,15 @@ _CODE_PATTERNS = {
re.compile(r"^\s*@\w+"), # annotations
re.compile(r"^\s*package\s+[\w.]+;"),
],
+ "csharp": [
+ re.compile(r"^\s*using\s+[\w.]+\s*;"), # using directive (not C++ `using namespace x;`)
+ re.compile(r"^\s*namespace\s+[\w.]+"),
+ re.compile(
+ r"^\s*(public|private|protected|internal|sealed|static|abstract|partial)\s+"
+ r"(class|struct|record|interface|enum)\b"
+ ),
+ re.compile(r"^.*\b(get|set|init);"), # auto-property accessors
+ ],
}
# Log/build output patterns
diff --git a/tests/test_transforms/test_code_compressor.py b/tests/test_transforms/test_code_compressor.py
index 059e35cac..574b438a1 100644
--- a/tests/test_transforms/test_code_compressor.py
+++ b/tests/test_transforms/test_code_compressor.py
@@ -869,6 +869,30 @@ class TestTreeSitterIntegration:
" e\n",
"}\n}",
),
+ (
+ "csharp",
+ (
+ "namespace Acme\n"
+ "{\n"
+ " public class Calc\n"
+ " {\n"
+ " public int Compute(int x)\n"
+ " {\n"
+ " int a = x + 1;\n"
+ " int b = a * 2;\n"
+ " int c = b - 3;\n"
+ " int d = c / 4;\n"
+ " int e = d + 5;\n"
+ " return e;\n"
+ " }\n"
+ " }\n"
+ "}\n"
+ ),
+ "public int Compute(int x)",
+ 5,
+ "return e;",
+ "}\n}",
+ ),
],
)
def test_compresses_methods_inside_class_member_containers(
@@ -1648,3 +1672,299 @@ class TestRealASTRuns:
assert result.syntax_valid is True
# Signature is preserved verbatim.
assert "pub fn add(a: i64, b: i64) -> i64" in result.compressed
+
+
[email protected](not TREE_SITTER_INSTALLED, reason="tree-sitter grammar pack not installed")
+class TestCSharpSupport:
+ """C# (``csharp`` grammar) parity with Java/C++/Rust: signatures preserved
+ verbatim, method/constructor bodies compressed, block-scoped namespaces
+ routed through container compression (no verbatim re-dump), file-scoped
+ namespace headers kept ahead of types, malformed input passed through.
+ """
+
+ def _compressor(self):
+ return CodeAwareCompressor(
+ CodeCompressorConfig(
+ min_tokens_for_compression=1,
+ max_body_lines=1,
+ enable_ccr=False,
+ )
+ )
+
+ def test_block_scoped_namespace_compresses_without_redumping(self):
+ """Block-scoped ``namespace { }`` wraps types in a declaration_list; the
+ container must compress its members and NOT be re-emitted verbatim
+ (which would duplicate the class and defeat compression — the C#
+ analogue of the #1318 class-member trap, one nesting level deeper)."""
+ code = (
+ "using System;\n"
+ "\n"
+ "namespace Acme.Widgets\n"
+ "{\n"
+ " public class WidgetService\n"
+ " {\n"
+ " public int Process(int x)\n"
+ " {\n"
+ " var a = x + 1;\n"
+ " var b = a + 2;\n"
+ " var c = b + 3;\n"
+ " Console.WriteLine(c);\n"
+ " return c;\n"
+ " }\n"
+ " }\n"
+ "}\n"
+ )
+ result = self._compressor().compress(code, language="csharp")
+
+ assert result.language == CodeLanguage.CSHARP
+ assert result.syntax_valid is True
+ assert result.compression_ratio < 1.0
+ # namespace + type headers preserved verbatim
+ assert "namespace Acme.Widgets" in result.compressed
+ assert "public class WidgetService" in result.compressed
+ assert "public int Process(int x)" in result.compressed
+ # method body actually compressed
+ assert "lines omitted" in result.compressed
+ assert "return c;" not in result.compressed
+ # the class is emitted exactly once (no re-dump / duplication)
+ assert result.compressed.count("class WidgetService") == 1
+ # block-namespace wrapper still closes
+ assert result.compressed.rstrip().endswith("}")
+
+ def test_file_scoped_namespace_directive_precedes_types(self):
+ """A file-scoped ``namespace X;`` must stay ahead of the type
+ declarations; emitting it after a type would not be valid C#."""
+ code = (
+ "using System;\n"
+ "\n"
+ "namespace Acme.Tools;\n"
+ "\n"
+ "public class Helper\n"
+ "{\n"
+ " public int Add(int a, int b)\n"
+ " {\n"
+ " var s = a + b;\n"
+ " var t = s + 1;\n"
+ " var u = t + 2;\n"
+ " return u;\n"
+ " }\n"
+ "}\n"
+ )
+ result = self._compressor().compress(code, language="csharp")
+
+ assert result.language == CodeLanguage.CSHARP
+ assert result.syntax_valid is True
+ assert result.compression_ratio < 1.0
+ assert "namespace Acme.Tools;" in result.compressed
+ assert "public class Helper" in result.compressed
+ assert result.compressed.index("namespace Acme.Tools;") < result.compressed.index(
+ "class Helper"
+ )
+
+ def test_record_preserves_positional_parameters(self):
+ """Record primary-constructor parameters are part of the signature and
+ must be preserved verbatim while the method body compresses."""
+ code = (
+ "namespace Acme.Models;\n"
+ "\n"
+ "public record Point(int X, int Y)\n"
+ "{\n"
+ " public double Dist()\n"
+ " {\n"
+ " var sq = X * X + Y * Y;\n"
+ " var r = System.Math.Sqrt(sq);\n"
+ " System.Console.WriteLine(r);\n"
+ " return r;\n"
+ " }\n"
+ "}\n"
+ )
+ result = self._compressor().compress(code, language="csharp")
+
+ assert result.language == CodeLanguage.CSHARP
+ assert result.syntax_valid is True
+ assert result.compression_ratio < 1.0
+ assert "public record Point(int X, int Y)" in result.compressed
+ assert "public double Dist()" in result.compressed
+ assert "lines omitted" in result.compressed
+
+ def test_preprocessor_wrapped_file_is_not_duplicated(self):
+ """A file wrapped in ``#if``/``#endif`` (idiomatic in multi-targeted
+ .NET code) must not have its content emitted twice. The visitor
+ recurses into ``preproc_if`` and captures the declarations, but the
+ top-level pass used to re-emit the uncaptured wrapper verbatim,
+ duplicating the whole file (observed on real repos: output up to
+ ~1.9x the input). Opaque handling keeps it verbatim exactly once."""
+ code = (
+ "#if !HAVE_TRACE_WRITER\n"
+ "using System;\n"
+ "\n"
+ "namespace Acme\n"
+ "{\n"
+ " public enum TraceLevel\n"
+ " {\n"
+ " Off = 0,\n"
+ " Error = 1,\n"
+ " }\n"
+ "}\n"
+ "#endif\n"
+ )
+ result = self._compressor().compress(code, language="csharp")
+
+ assert result.syntax_valid is True
+ # Content appears exactly once — no wrapper re-dump.
+ assert result.compressed.count("enum TraceLevel") == 1
+ assert result.compressed.count("using System;") == 1
+ # Never larger than the input (verbatim pass-through is acceptable).
+ assert result.compression_ratio <= 1.0
+ # Conditional directives stay balanced.
+ assert result.compressed.count("#if") == result.compressed.count("#endif")
+
+ def test_preprocessor_wrapped_usings_stay_ahead_of_types(self):
+ """``#if``-wrapped using directives (idiomatic for multi-targeting)
+ must be emitted with the imports. Appending them as trailing top-level
+ code puts usings after type declarations — invalid C#, which trips the
+ output syntax check and falls back to no compression at all (observed
+ on real repos: hundreds of files silently uncompressed)."""
+ code = (
+ "using System;\n"
+ "#if HAVE_BIG_INTEGER\n"
+ "using System.Numerics;\n"
+ "#endif\n"
+ "\n"
+ "namespace Acme\n"
+ "{\n"
+ " public class Calc\n"
+ " {\n"
+ " public int Compute(int x)\n"
+ " {\n"
+ " var a = x + 1;\n"
+ " var b = a * 2;\n"
+ " var c = b - 3;\n"
+ " var d = c / 4;\n"
+ " return d;\n"
+ " }\n"
+ " }\n"
+ "}\n"
+ )
+ result = self._compressor().compress(code, language="csharp")
+
+ assert result.syntax_valid is True
+ # Compression must actually happen (no silent fallback).
+ assert result.compression_ratio < 1.0
+ assert "lines omitted" in result.compressed
+ # The conditional import block survives verbatim, ahead of the types.
+ assert "#if HAVE_BIG_INTEGER" in result.compressed
+ assert "using System.Numerics;" in result.compressed
+ assert result.compressed.index("#endif") < result.compressed.index("class Calc")
+
+ def test_region_markers_inside_class_compress_cleanly(self):
+ """``#region``/``#endregion`` markers span their trailing newline, so
+ their end_point sits at column 0 of the NEXT line. Line-based child
+ extraction used to swallow that line — duplicating the following
+ method's signature and the class's closing brace (invalid output →
+ silent fallback to no compression on real repos)."""
+ code = (
+ "namespace Acme\n"
+ "{\n"
+ " public class Svc\n"
+ " {\n"
+ " #region Api\n"
+ " public int F(int x)\n"
+ " {\n"
+ " var a = x + 1;\n"
+ " var b = a * 2;\n"
+ " var c = b - 3;\n"
+ " return c;\n"
+ " }\n"
+ " #endregion\n"
+ " }\n"
+ "}\n"
+ )
+ result = self._compressor().compress(code, language="csharp")
+
+ assert result.syntax_valid is True
+ assert result.compression_ratio < 1.0
+ assert "lines omitted" in result.compressed
+ # Signature emitted exactly once (regions used to duplicate it).
+ assert result.compressed.count("public int F(int x)") == 1
+ # Region markers survive, braces stay balanced.
+ assert "#region Api" in result.compressed
+ assert "#endregion" in result.compressed
+ assert result.compressed.count("{") == result.compressed.count("}")
+
+ def test_license_region_header_stays_on_top(self):
+ """A ``#region License`` banner above the usings (near-universal in
+ real .NET code) must stay at the top of the file. Relocating it after
+ the type declarations — the old behavior for uncaptured top-level
+ nodes — is rejected by tree-sitter-c-sharp, failing output validation
+ and silently forfeiting compression for the whole file."""
+ code = (
+ "#region License\n"
+ "// Copyright (c) 2007 Example Corp.\n"
+ "// Licensed under the MIT license.\n"
+ "#endregion\n"
+ "\n"
+ "using System;\n"
+ "\n"
+ "namespace Acme\n"
+ "{\n"
+ " public class Svc\n"
+ " {\n"
+ " public int F(int x)\n"
+ " {\n"
+ " var a = x + 1;\n"
+ " var b = a * 2;\n"
+ " var c = b - 3;\n"
+ " return c;\n"
+ " }\n"
+ " }\n"
+ "}\n"
+ )
+ result = self._compressor().compress(code, language="csharp")
+
+ assert result.syntax_valid is True
+ assert result.compression_ratio < 1.0
+ assert "lines omitted" in result.compressed
+ # Header block survives, in order, ahead of everything else.
+ assert result.compressed.index("#region License") < result.compressed.index("using System;")
+ assert result.compressed.index("// Copyright") < result.compressed.index("#endregion")
+ assert result.compressed.index("#endregion") < result.compressed.index("class Svc")
+
+ def test_malformed_csharp_passes_through_unchanged(self):
+ """Malformed C# must pass through unchanged (no data loss; prefer false
+ negatives over serving broken or altered code)."""
+ code = (
+ "namespace Broken\n"
+ "{\n"
+ " public class Oops\n"
+ " {\n"
+ " public int F(int x)\n"
+ " {\n"
+ " var a = x + 1\n" # missing ';' and unclosed braces
+ " return a\n"
+ )
+ result = self._compressor().compress(code, language="csharp")
+
+ assert result.compressed == code
+ assert result.compression_ratio == 1.0
+
+ def test_detect_language_identifies_csharp(self):
+ """Auto-detection (no explicit language hint) recognizes C#."""
+ code = (
+ "using System;\n"
+ "\n"
+ "namespace Acme\n"
+ "{\n"
+ " public class Svc\n"
+ " {\n"
+ " public int Total { get; set; }\n"
+ " public int Add(int a, int b)\n"
+ " {\n"
+ " return a + b;\n"
+ " }\n"
+ " }\n"
+ "}\n"
+ )
+ lang, confidence = detect_language(code)
+ assert lang == CodeLanguage.CSHARP
+ assert confidence > 0.0