Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(compression): preserve Bash control flow (#3425)
宇宙的山峰ee902074a04d
diff --git a/headroom/transforms/code_compressor.py b/headroom/transforms/code_compressor.pyindex d95b7265e..ef9b8379a 100644--- a/headroom/transforms/code_compressor.py+++ b/headroom/transforms/code_compressor.py@@ -17,6 +17,7 @@ Supported Languages (Tier 1):Supported Languages (Tier 2):- Go, Rust, Java, C, C+++- Bash (validated and preserved losslessly)Compression Strategy:1. Parse code into AST using tree-sitter@@ -273,7 +274,7 @@ def _get_parser(language: str) -> Any:except Exception as e:raise ValueError(f"Language '{language}' is not supported by tree-sitter. "- f"Supported: python, javascript, typescript, go, rust, java, c, cpp, csharp, php. "+ f"Supported: python, javascript, typescript, go, rust, java, c, cpp, csharp, php, bash. "f"Error: {e}") from e@@ -330,6 +331,8 @@ class CodeLanguage(Enum):PERL = "perl"CSHARP = "csharp"PHP = "php"+ BASH = "bash"+ SHELL = "bash" # Alias: tree-sitter-language-pack exposes the Bash grammar.UNKNOWN = "unknown"@@ -358,6 +361,10 @@ _LANGUAGE_ALIASES: dict[str, CodeLanguage] = {"php5": CodeLanguage.PHP,"php7": CodeLanguage.PHP,"php8": CodeLanguage.PHP,+ "shell": CodeLanguage.BASH,+ "sh": CodeLanguage.BASH,+ "zsh": CodeLanguage.BASH,+ "shellscript": CodeLanguage.BASH,}@@ -589,6 +596,34 @@ _LANG_CONFIGS: dict[CodeLanguage, LangConfig] = {detection_hints=("<?php", "function ", "namespace ", "->", "$this"),class_body_node_types=frozenset({"declaration_list"}),),+ CodeLanguage.BASH: LangConfig(+ # Shell control-flow nodes are opaque: their `then`/`fi`, `do`/`done`,+ # and `case` delimiters are anonymous grammar tokens. The generic+ # extractor must not descend into them and re-emit only their commands,+ # which would silently corrupt otherwise valid shell code.+ import_nodes=frozenset(),+ function_nodes=frozenset(),+ class_nodes=frozenset(),+ type_nodes=frozenset(),+ body_node_types=frozenset(),+ decorator_node=None,+ comment_prefix="#",+ uses_colon_after_signature=False,+ detection_hints=("#!/bin/bash", "#!/usr/bin/env bash", " then", " fi", " esac"),+ opaque_node_types=frozenset(+ {+ "if_statement",+ "for_statement",+ "while_statement",+ "until_statement",+ "case_statement",+ "select_statement",+ "function_definition",+ "subshell",+ "compound_statement",+ }+ ),+ ),}@@ -804,6 +839,18 @@ _LANGUAGE_PREFILTER: dict[CodeLanguage, list[re.Pattern[str]]] = {),re.compile(r"\$this->|->\w+\s*\(", re.MULTILINE),],+ CodeLanguage.BASH: [+ re.compile(r"^\s*#!.*\b(?:bash|sh|zsh)\b", re.MULTILINE),+ re.compile(+ r"^\s*(?:if|then|elif|else|fi|for|while|until|do|done|case|esac|select)\b",+ re.MULTILINE,+ ),+ re.compile(+ r"^\s*(?:export|source|shopt|declare|local|alias|set|readonly|typeset)\b",+ re.MULTILINE,+ ),+ re.compile(r"\[\[.*\]\]|\$\{[^}]+\}|\$\([^)]+\)", re.MULTILINE),+ ],}@@ -1349,6 +1396,24 @@ class CodeAwareCompressor(Transform):syntax_valid=True,)+ # Bash control-flow is recognized and parsed for validation, but is not+ # rewritten yet. Its grammar uses anonymous delimiters (`then`, `fi`,+ # `do`, `done`, `esac`) that a generic AST reassembler cannot safely+ # preserve. Returning the validated source keeps shell scripts+ # byte-for-byte intact instead of sending them to lossy Kompress.+ if detected_lang == CodeLanguage.BASH:+ syntax_valid = self._verify_syntax(code, detected_lang)+ return CodeCompressionResult(+ compressed=code,+ original=code,+ original_tokens=original_tokens,+ compressed_tokens=original_tokens,+ compression_ratio=1.0,+ language=detected_lang,+ language_confidence=confidence,+ syntax_valid=syntax_valid,+ )+# Parse and compresstry:compressed, structure, symbol_scores = self._compress_with_ast(diff --git a/tests/test_code_compressor_bash.py b/tests/test_code_compressor_bash.pynew file mode 100644index 000000000..1da57bcd9--- /dev/null+++ b/tests/test_code_compressor_bash.py@@ -0,0 +1,69 @@+"""Regression tests for lossless Bash handling in CodeAwareCompressor."""++from headroom.transforms.code_compressor import (+ CodeAwareCompressor,+ CodeCompressorConfig,+ CodeLanguage,+ coerce_language,+ detect_language,+)++BASH_SNIPPET = """#!/usr/bin/env bash+set -euo pipefail++if [ -x "$(command -v /opt/bin/forgejo)" ]+then+\texport GITEA_WORK_DIR=/git/forgejo+\texport FORGEJO_CUSTOM=/etc/forgejo+\talias forge="sudo -Eu git GITEA_CUSTOM=${FORGEJO_CUSTOM} GITEA_WORK_DIR=${GITEA_WORK_DIR} /opt/bin/forgejo"+fi++if [[ "$( git config --global alias.pushall 2>/dev/null )" != '!git remote | xargs -L1 git push --all' ]]+then+\tgit config --global --unset-all alias.pushall+git config --global --add alias.pushall '!git remote | xargs -L1 git push --all'+fi+"""+++def test_bash_aliases_are_canonicalized():+ assert coerce_language("bash") is CodeLanguage.BASH+ assert coerce_language("shell") is CodeLanguage.BASH+ assert coerce_language("sh") is CodeLanguage.BASH+ assert coerce_language("zsh") is CodeLanguage.BASH+++def test_bash_is_detected_from_control_flow():+ detected, confidence = detect_language(BASH_SNIPPET.removeprefix("#!/usr/bin/env bash\n"))+ assert detected is CodeLanguage.BASH+ assert confidence > 0+++def test_bash_is_detected_from_shebang():+ detected, confidence = detect_language("#!/bin/sh\necho hello\n")+ assert detected is CodeLanguage.BASH+ assert confidence > 0+++def test_bash_compression_is_lossless_and_does_not_use_kompress():+ compressor = CodeAwareCompressor(+ CodeCompressorConfig(min_tokens_for_compression=1, enable_ccr=False)+ )+ result = compressor.compress(BASH_SNIPPET)++ assert result.language is CodeLanguage.BASH+ assert result.syntax_valid is True+ assert result.compressed == BASH_SNIPPET+ assert result.compressed_tokens == result.original_tokens+ assert result.compression_ratio == 1.0+++def test_shell_hint_is_lossless():+ compressor = CodeAwareCompressor(+ CodeCompressorConfig(min_tokens_for_compression=1, enable_ccr=False)+ )+ result = compressor.compress(BASH_SNIPPET, language="shell")++ assert result.language is CodeLanguage.BASH+ assert result.compressed == BASH_SNIPPET+ assert result.syntax_valid is True