Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(compression): preserve Bash control flow (#3425)

宇宙的山峰ee902074a04d
diff --git a/headroom/transforms/code_compressor.py b/headroom/transforms/code_compressor.py
index d95b7265e..ef9b8379a 100644
--- a/headroom/transforms/code_compressor.py
+++ b/headroom/transforms/code_compressor.py
@@ -17,6 +17,7 @@ Supported Languages (Tier 1):
Supported Languages (Tier 2):
- Go, Rust, Java, C, C++
+- Bash (validated and preserved losslessly)
Compression Strategy:
1. Parse code into AST using tree-sitter
@@ -273,7 +274,7 @@ def _get_parser(language: str) -> Any:
except Exception as e:
raise ValueError(
f"Language '{language}' is not supported by tree-sitter. "
- f"Supported: python, javascript, typescript, go, rust, java, c, cpp, csharp, php. "
+ f"Supported: python, javascript, typescript, go, rust, java, c, cpp, csharp, php, bash. "
f"Error: {e}"
) from e
@@ -330,6 +331,8 @@ class CodeLanguage(Enum):
PERL = "perl"
CSHARP = "csharp"
PHP = "php"
+ BASH = "bash"
+ SHELL = "bash" # Alias: tree-sitter-language-pack exposes the Bash grammar.
UNKNOWN = "unknown"
@@ -358,6 +361,10 @@ _LANGUAGE_ALIASES: dict[str, CodeLanguage] = {
"php5": CodeLanguage.PHP,
"php7": CodeLanguage.PHP,
"php8": CodeLanguage.PHP,
+ "shell": CodeLanguage.BASH,
+ "sh": CodeLanguage.BASH,
+ "zsh": CodeLanguage.BASH,
+ "shellscript": CodeLanguage.BASH,
}
@@ -589,6 +596,34 @@ _LANG_CONFIGS: dict[CodeLanguage, LangConfig] = {
detection_hints=("<?php", "function ", "namespace ", "->", "$this"),
class_body_node_types=frozenset({"declaration_list"}),
),
+ CodeLanguage.BASH: LangConfig(
+ # Shell control-flow nodes are opaque: their `then`/`fi`, `do`/`done`,
+ # and `case` delimiters are anonymous grammar tokens. The generic
+ # extractor must not descend into them and re-emit only their commands,
+ # which would silently corrupt otherwise valid shell code.
+ import_nodes=frozenset(),
+ function_nodes=frozenset(),
+ class_nodes=frozenset(),
+ type_nodes=frozenset(),
+ body_node_types=frozenset(),
+ decorator_node=None,
+ comment_prefix="#",
+ uses_colon_after_signature=False,
+ detection_hints=("#!/bin/bash", "#!/usr/bin/env bash", " then", " fi", " esac"),
+ opaque_node_types=frozenset(
+ {
+ "if_statement",
+ "for_statement",
+ "while_statement",
+ "until_statement",
+ "case_statement",
+ "select_statement",
+ "function_definition",
+ "subshell",
+ "compound_statement",
+ }
+ ),
+ ),
}
@@ -804,6 +839,18 @@ _LANGUAGE_PREFILTER: dict[CodeLanguage, list[re.Pattern[str]]] = {
),
re.compile(r"\$this->|->\w+\s*\(", re.MULTILINE),
],
+ CodeLanguage.BASH: [
+ re.compile(r"^\s*#!.*\b(?:bash|sh|zsh)\b", re.MULTILINE),
+ re.compile(
+ r"^\s*(?:if|then|elif|else|fi|for|while|until|do|done|case|esac|select)\b",
+ re.MULTILINE,
+ ),
+ re.compile(
+ r"^\s*(?:export|source|shopt|declare|local|alias|set|readonly|typeset)\b",
+ re.MULTILINE,
+ ),
+ re.compile(r"\[\[.*\]\]|\$\{[^}]+\}|\$\([^)]+\)", re.MULTILINE),
+ ],
}
@@ -1349,6 +1396,24 @@ class CodeAwareCompressor(Transform):
syntax_valid=True,
)
+ # Bash control-flow is recognized and parsed for validation, but is not
+ # rewritten yet. Its grammar uses anonymous delimiters (`then`, `fi`,
+ # `do`, `done`, `esac`) that a generic AST reassembler cannot safely
+ # preserve. Returning the validated source keeps shell scripts
+ # byte-for-byte intact instead of sending them to lossy Kompress.
+ if detected_lang == CodeLanguage.BASH:
+ syntax_valid = self._verify_syntax(code, detected_lang)
+ return CodeCompressionResult(
+ compressed=code,
+ original=code,
+ original_tokens=original_tokens,
+ compressed_tokens=original_tokens,
+ compression_ratio=1.0,
+ language=detected_lang,
+ language_confidence=confidence,
+ syntax_valid=syntax_valid,
+ )
+
# Parse and compress
try:
compressed, structure, symbol_scores = self._compress_with_ast(
diff --git a/tests/test_code_compressor_bash.py b/tests/test_code_compressor_bash.py
new file mode 100644
index 000000000..1da57bcd9
--- /dev/null
+++ b/tests/test_code_compressor_bash.py
@@ -0,0 +1,69 @@
+"""Regression tests for lossless Bash handling in CodeAwareCompressor."""
+
+from headroom.transforms.code_compressor import (
+ CodeAwareCompressor,
+ CodeCompressorConfig,
+ CodeLanguage,
+ coerce_language,
+ detect_language,
+)
+
+BASH_SNIPPET = """#!/usr/bin/env bash
+set -euo pipefail
+
+if [ -x "$(command -v /opt/bin/forgejo)" ]
+then
+\texport GITEA_WORK_DIR=/git/forgejo
+\texport FORGEJO_CUSTOM=/etc/forgejo
+\talias forge="sudo -Eu git GITEA_CUSTOM=${FORGEJO_CUSTOM} GITEA_WORK_DIR=${GITEA_WORK_DIR} /opt/bin/forgejo"
+fi
+
+if [[ "$( git config --global alias.pushall 2>/dev/null )" != '!git remote | xargs -L1 git push --all' ]]
+then
+\tgit config --global --unset-all alias.pushall
+git config --global --add alias.pushall '!git remote | xargs -L1 git push --all'
+fi
+"""
+
+
+def test_bash_aliases_are_canonicalized():
+ assert coerce_language("bash") is CodeLanguage.BASH
+ assert coerce_language("shell") is CodeLanguage.BASH
+ assert coerce_language("sh") is CodeLanguage.BASH
+ assert coerce_language("zsh") is CodeLanguage.BASH
+
+
+def test_bash_is_detected_from_control_flow():
+ detected, confidence = detect_language(BASH_SNIPPET.removeprefix("#!/usr/bin/env bash\n"))
+ assert detected is CodeLanguage.BASH
+ assert confidence > 0
+
+
+def test_bash_is_detected_from_shebang():
+ detected, confidence = detect_language("#!/bin/sh\necho hello\n")
+ assert detected is CodeLanguage.BASH
+ assert confidence > 0
+
+
+def test_bash_compression_is_lossless_and_does_not_use_kompress():
+ compressor = CodeAwareCompressor(
+ CodeCompressorConfig(min_tokens_for_compression=1, enable_ccr=False)
+ )
+ result = compressor.compress(BASH_SNIPPET)
+
+ assert result.language is CodeLanguage.BASH
+ assert result.syntax_valid is True
+ assert result.compressed == BASH_SNIPPET
+ assert result.compressed_tokens == result.original_tokens
+ assert result.compression_ratio == 1.0
+
+
+def test_shell_hint_is_lossless():
+ compressor = CodeAwareCompressor(
+ CodeCompressorConfig(min_tokens_for_compression=1, enable_ccr=False)
+ )
+ result = compressor.compress(BASH_SNIPPET, language="shell")
+
+ assert result.language is CodeLanguage.BASH
+ assert result.compressed == BASH_SNIPPET
+ assert result.syntax_valid is True