Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Fix SmartCrusher bugs: schema violation, race condition, thread safety, recursion
chopratejas14415dbbb56c
diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.pyindex 97c0e0891..5d354a621 100644--- a/headroom/transforms/smart_crusher.py+++ b/headroom/transforms/smart_crusher.py@@ -180,27 +180,31 @@ def _hash_field_name(field_name: str) -> str:# Minimum chars for a text field to be worth compressing within an item_MIN_FIELD_CHARS_FOR_WITHIN = 200-# Lazy-loaded compressor for within-item text compression+# Lazy-loaded compressor for within-item text compression (thread-safe)_within_compressor: Any = None_within_compressor_checked = False+_within_compressor_lock = threading.Lock()def _get_within_compressor() -> Any:"""Get a text compressor for within-item field compression.Returns Kompress if available (requires [ml] extra), else None.+ Thread-safe via double-checked locking."""global _within_compressor, _within_compressor_checkedif not _within_compressor_checked:- _within_compressor_checked = True- try:- from .kompress_compressor import KompressCompressor, is_kompress_available-- if is_kompress_available():- _within_compressor = KompressCompressor()- logger.debug("Within-item compression: using Kompress")- except ImportError:- pass+ with _within_compressor_lock:+ if not _within_compressor_checked:+ try:+ from .kompress_compressor import KompressCompressor, is_kompress_available++ if is_kompress_available():+ _within_compressor = KompressCompressor()+ logger.debug("Within-item compression: using Kompress")+ except ImportError:+ pass+ _within_compressor_checked = Truereturn _within_compressor@@ -435,7 +439,7 @@ def _detect_sequential_pattern(values: list[Any], check_order: bool = True) -> b# Get numeric valuesnums = []for v in values:- if isinstance(v, (int, float)) and not isinstance(v, bool):+ if isinstance(v, int | float) and not isinstance(v, bool):nums.append(v)elif isinstance(v, str):try:@@ -546,7 +550,6 @@ def _detect_score_field_statistically(stats: FieldStats, items: list[dict]) -> tconfidence = 0.0# Check for bounded range typical of scores- stats.max_val - stats.min_valmin_val, max_val = stats.min_val, stats.max_val# Common score ranges: [0,1], [0,10], [0,100], [-1,1], [0,5]@@ -578,7 +581,7 @@ def _detect_score_field_statistically(stats: FieldStats, items: list[dict]) -> tfor item in items:if stats.name in item:val = item.get(stats.name)- if isinstance(val, (int, float)) and math.isfinite(val):+ if isinstance(val, int | float) and math.isfinite(val):values_in_order.append(float(val))if len(values_in_order) >= 5:# Check for descending sort@@ -804,11 +807,11 @@ def _detect_items_by_learned_semantics(value_canonical = "null"elif isinstance(value, bool):value_canonical = "true" if value else "false"- elif isinstance(value, (int, float)):+ elif isinstance(value, int | float):value_canonical = str(value)elif isinstance(value, str):value_canonical = value- elif isinstance(value, (list, dict)):+ elif isinstance(value, list | dict):try:value_canonical = json.dumps(value, sort_keys=True, default=str)except (TypeError, ValueError):@@ -1030,7 +1033,7 @@ class SmartAnalyzer:first_val = non_null_values[0]if isinstance(first_val, bool):field_type = "boolean"- elif isinstance(first_val, (int, float)):+ elif isinstance(first_val, int | float):field_type = "numeric"elif isinstance(first_val, str):field_type = "string"@@ -1064,7 +1067,7 @@ class SmartAnalyzer:# Numeric-specific analysisif field_type == "numeric":# Filter out NaN and Infinity which break statistics functions- nums = [v for v in non_null_values if isinstance(v, (int, float)) and math.isfinite(v)]+ nums = [v for v in non_null_values if isinstance(v, int | float) and math.isfinite(v)]if nums:try:stats.min_val = min(nums)@@ -1283,7 +1286,7 @@ class SmartAnalyzer:threshold = self.config.variance_threshold * stdfor i, item in enumerate(items):val = item.get(stats.name)- if isinstance(val, (int, float)):+ if isinstance(val, int | float):if abs(val - stats.mean_val) > threshold:anomaly_indices.add(i)@@ -1953,9 +1956,9 @@ class SmartCrusher(Transform):if len(keep_indices) <= effective_max:return keep_indices- # Use provided field_semantics or fall back to instance variable (set by crush())+ # Use provided field_semantics or fall back to thread-local (set by _crush_array)effective_field_semantics = field_semantics or getattr(- self, "_current_field_semantics", None+ getattr(self, "_thread_local", None), "field_semantics", None)# Identify error items using KEYWORD detection (preservation guarantee)@@ -1976,7 +1979,7 @@ class SmartCrusher(Transform):threshold = self.config.variance_threshold * stdfor i, item in enumerate(items):val = item.get(field_name)- if isinstance(val, (int, float)):+ if isinstance(val, int | float):if abs(val - stats.mean_val) > threshold:anomaly_indices.add(i)@@ -2297,6 +2300,10 @@ class SmartCrusher(Transform):return result, was_modified, info+ # Maximum recursion depth for nested JSON processing.+ # Prevents RecursionError on adversarial/deeply-nested input.+ _MAX_PROCESS_DEPTH = 50+def _process_value(self,value: Any,@@ -2311,6 +2318,10 @@ class SmartCrusher(Transform):Tuple of (processed_value, info_string, ccr_markers).ccr_markers is a list of (hash, original_count, compressed_count, summary) tuples."""+ # Guard against deeply nested JSON causing RecursionError+ if depth >= self._MAX_PROCESS_DEPTH:+ return value, "", []+info_parts = []ccr_markers: list[tuple] = []@@ -2495,9 +2506,12 @@ class SmartCrusher(Transform):)# === TOIN Evolution: Extract field semantics for signal detection ===- # Store temporarily on instance for use in _prioritize_indices+ # Store in thread-local storage for use in _prioritize_indices.# This enables learned signal detection without changing all method signatures- self._current_field_semantics = (+ # while remaining thread-safe (no cross-thread contamination).+ if not hasattr(self, "_thread_local"):+ self._thread_local = threading.local()+ self._thread_local.field_semantics = (toin_hint.field_semantics if toin_hint.field_semantics else None)@@ -2661,12 +2675,14 @@ class SmartCrusher(Transform):)# Clean up temporary instance variable- self._current_field_semantics = None+ if hasattr(self, "_thread_local"):+ self._thread_local.field_semantics = Nonereturn result, strategy_info, ccr_hash, dropped_summaryexcept Exception:# Clean up temporary instance variable- self._current_field_semantics = None+ if hasattr(self, "_thread_local"):+ self._thread_local.field_semantics = None# Re-raise any exceptions (removed finally block since we no longer mutate config)raise@@ -2814,7 +2830,7 @@ class SmartCrusher(Transform):return items, "number:passthrough"# Filter out non-finite values for statistics- finite = [x for x in items if isinstance(x, (int, float)) and math.isfinite(x)]+ finite = [x for x in items if isinstance(x, int | float) and math.isfinite(x)]if not finite:return items, "number:no_finite"@@ -2832,7 +2848,7 @@ class SmartCrusher(Transform):outlier_indices: set[int] = set()if std_val > 0:for i, val in enumerate(items):- if isinstance(val, (int, float)) and math.isfinite(val):+ if isinstance(val, int | float) and math.isfinite(val):if abs(val - mean_val) > self.config.variance_threshold * std_val:outlier_indices.add(i)@@ -2844,12 +2860,12 @@ class SmartCrusher(Transform):left = [items[j]for j in range(i - window, i)- if isinstance(items[j], (int, float)) and math.isfinite(items[j])+ if isinstance(items[j], int | float) and math.isfinite(items[j])]right = [items[j]for j in range(i, i + window)- if isinstance(items[j], (int, float)) and math.isfinite(items[j])+ if isinstance(items[j], int | float) and math.isfinite(items[j])]if left and right:left_mean = statistics.mean(left)@@ -2877,27 +2893,23 @@ class SmartCrusher(Transform):if i not in keep_indices:keep_indices.add(i)- # Build output: summary string + kept values in original order- stats_summary = (- f"[{n} numbers: min={min(finite)}, max={max(finite)}, "- f"mean={mean_val:.4g}, median={median_val:.4g}, "- f"stddev={std_val:.4g}, p25={p25:.4g}, p75={p75:.4g}"- )- if outlier_indices:- stats_summary += f", outliers={len(outlier_indices)}"- if change_indices:- stats_summary += f", change_points={len(change_indices)}"- stats_summary += "]"-+ # Build output: kept values only (schema-preserving — no generated text)kept_values = [items[i] for i in sorted(keep_indices)]- result: list = [stats_summary] + kept_values- strategy = f"number:adaptive({n}->{len(kept_values)}"+ # Encode statistics into the strategy string (not the array itself)+ strategy = (+ f"number:adaptive({n}->{len(kept_values)}"+ f",min={min(finite)},max={max(finite)}"+ f",mean={mean_val:.4g},median={median_val:.4g}"+ f",stddev={std_val:.4g},p25={p25:.4g},p75={p75:.4g}"+ )if outlier_indices:strategy += f",outliers={len(outlier_indices)}"+ if change_indices:+ strategy += f",change_points={len(change_indices)}"strategy += ")"- return result, strategy+ return kept_values, strategydef _crush_mixed_array(self,@@ -2930,7 +2942,7 @@ class SmartCrusher(Transform):key = "str"elif isinstance(item, bool):key = "bool"- elif isinstance(item, (int, float)):+ elif isinstance(item, int | float):key = "number"elif isinstance(item, list):key = "list"@@ -2979,13 +2991,13 @@ class SmartCrusher(Transform):last_idx = set(indices[-k_last:])keep_indices.update(first_idx | last_idx)# Outliers- finite = [v for v in values if isinstance(v, (int, float)) and math.isfinite(v)]+ finite = [v for v in values if isinstance(v, int | float) and math.isfinite(v)]if len(finite) > 1:mean_v = statistics.mean(finite)std_v = statistics.stdev(finite)if std_v > 0:for idx, val in group_items:- if isinstance(val, (int, float)) and math.isfinite(val):+ if isinstance(val, int | float) and math.isfinite(val):if abs(val - mean_v) > self.config.variance_threshold * std_v:keep_indices.add(idx)strategy_parts.append(f"num:{len(values)}")@@ -3553,7 +3565,7 @@ class SmartCrusher(Transform):threshold = self.config.variance_threshold * stdfor i, item in enumerate(items):val = item.get(name)- if isinstance(val, (int, float)):+ if isinstance(val, int | float):if abs(val - stats.mean_val) > threshold:keep_indices.add(i)diff --git a/tests/test_transforms/test_smart_crusher_bugs.py b/tests/test_transforms/test_smart_crusher_bugs.pynew file mode 100644index 000000000..00310dc40--- /dev/null+++ b/tests/test_transforms/test_smart_crusher_bugs.py@@ -0,0 +1,212 @@+"""Regression tests for SmartCrusher bugs.++Bug 1: _crush_number_array mixes types (string summary + numbers),+ violating the schema-preserving guarantee.+Bug 2: _current_field_semantics is shared instance state, creating+ a race condition when crushing concurrently.+"""++from __future__ import annotations++import json+from concurrent.futures import ThreadPoolExecutor, as_completed++from headroom import SmartCrusherConfig+from headroom.transforms.smart_crusher import SmartCrusher++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------+++def _make_crusher(max_items: int = 10, min_items: int = 3) -> SmartCrusher:+ config = SmartCrusherConfig(+ enabled=True,+ min_items_to_analyze=min_items,+ min_tokens_to_crush=0,+ max_items_after_crush=max_items,+ variance_threshold=2.0,+ )+ return SmartCrusher(config=config)+++# ---------------------------------------------------------------------------+# Bug 1: Number array type mixing+# ---------------------------------------------------------------------------+++class TestNumberArraySchemaPreservation:+ """_crush_number_array must return only original numeric values.++ Previously it prepended a stats summary string, producing+ [string, int, int, ...] which violates the schema-preserving+ guarantee and breaks type-aware JSON consumers.+ """++ def test_crushed_number_array_contains_only_numbers(self) -> None:+ """Every element of the crushed array must be int or float."""+ crusher = _make_crusher(max_items=10)+ numbers = list(range(50)) # 0..49, well above the n<=8 passthrough+ crushed, strategy = crusher._crush_number_array(numbers)++ for i, item in enumerate(crushed):+ assert isinstance(item, int | float), (+ f"Item {i} is {type(item).__name__} = {item!r}, expected int/float. "+ f"Schema-preserving guarantee violated."+ )++ def test_crushed_number_array_subset_of_original(self) -> None:+ """Every value in the crushed array must exist in the original."""+ crusher = _make_crusher(max_items=10)+ numbers = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]+ crushed, _ = crusher._crush_number_array(numbers)++ original_set = set(numbers)+ for item in crushed:+ assert item in original_set, (+ f"Value {item!r} not in original array — generated content detected"+ )++ def test_stats_summary_in_strategy_not_in_array(self) -> None:+ """Statistics should be communicated via strategy string, not array content."""+ crusher = _make_crusher(max_items=5)+ numbers = list(range(100))+ crushed, strategy = crusher._crush_number_array(numbers)++ # Strategy should contain stats info+ assert "number:" in strategy++ # Array should not contain any strings+ strings_in_result = [x for x in crushed if isinstance(x, str)]+ assert strings_in_result == [], f"Found string(s) in numeric array: {strings_in_result}"++ def test_number_array_passthrough_for_small(self) -> None:+ """Arrays with n <= 8 should pass through unchanged."""+ crusher = _make_crusher()+ small = [1, 2, 3, 4, 5]+ crushed, strategy = crusher._crush_number_array(small)+ assert crushed == small+ assert strategy == "number:passthrough"++ def test_number_array_preserves_outliers(self) -> None:+ """Outlier values should be preserved in the crushed output."""+ crusher = _make_crusher(max_items=10)+ # Normal range + extreme outlier+ numbers = [10] * 20 + [10000]+ crushed, strategy = crusher._crush_number_array(numbers)+ assert 10000 in crushed, "Outlier value 10000 was dropped"++ def test_number_array_preserves_boundaries(self) -> None:+ """First and last values should always be kept."""+ crusher = _make_crusher(max_items=5)+ numbers = list(range(100))+ crushed, strategy = crusher._crush_number_array(numbers)+ assert crushed[0] == 0, "First value not preserved"+ assert numbers[-1] in crushed, "Last value not preserved"++ def test_non_finite_passthrough(self) -> None:+ """All-NaN/Inf arrays should return unchanged."""+ crusher = _make_crusher()+ nans = [float("nan")] * 10+ crushed, strategy = crusher._crush_number_array(nans)+ assert strategy == "number:no_finite"+ assert len(crushed) == 10++ def test_full_crush_pipeline_number_array_types(self) -> None:+ """End-to-end: crushing a JSON number array via the public API."""+ crusher = _make_crusher(max_items=10)+ content = json.dumps(list(range(50)))+ result, was_modified, info = crusher._smart_crush_content(content)++ if was_modified:+ parsed = json.loads(result)+ assert isinstance(parsed, list)+ for item in parsed:+ assert isinstance(item, int | float), (+ f"Public API returned non-numeric item {item!r} in number array"+ )+++# ---------------------------------------------------------------------------+# Bug 2: Race condition on _current_field_semantics+# ---------------------------------------------------------------------------+++class TestFieldSemanticsThreadSafety:+ """_current_field_semantics must not leak between concurrent crushes.++ Previously it was stored as instance state (self._current_field_semantics)+ which created a race condition when the same SmartCrusher instance+ was used from multiple threads.+ """++ def test_concurrent_crushes_no_cross_contamination(self) -> None:+ """Two concurrent crushes must not share field_semantics state."""+ crusher = _make_crusher(max_items=5)++ # Two different array payloads+ payload_a = json.dumps([{"name": f"item_{i}", "value": i} for i in range(20)])+ payload_b = json.dumps([{"key": f"k_{i}", "score": i * 0.1} for i in range(20)])++ results: dict[str, str] = {}+ errors: list[Exception] = []++ def crush_task(label: str, content: str) -> None:+ try:+ result, modified, info = crusher._smart_crush_content(content)+ results[label] = result+ except Exception as e:+ errors.append(e)++ with ThreadPoolExecutor(max_workers=4) as executor:+ futures = []+ # Run many concurrent crushes to increase race probability+ for i in range(20):+ futures.append(executor.submit(crush_task, f"a_{i}", payload_a))+ futures.append(executor.submit(crush_task, f"b_{i}", payload_b))+ for f in as_completed(futures):+ f.result() # Re-raise exceptions++ assert not errors, f"Concurrent crushes raised errors: {errors}"++ # After all crushes, thread-local state must be clean+ tl = getattr(crusher, "_thread_local", None)+ if tl is not None:+ semantics = getattr(tl, "field_semantics", None)+ assert semantics is None, f"field_semantics leaked in thread-local: {semantics}"+++# ---------------------------------------------------------------------------+# Issue 7: Recursion depth limit+# ---------------------------------------------------------------------------+++class TestRecursionDepthLimit:+ """_process_value must not crash on deeply nested JSON."""++ def test_deeply_nested_json_does_not_crash(self) -> None:+ """Nesting deeper than _MAX_PROCESS_DEPTH should return value unchanged."""+ crusher = _make_crusher()+ # Build a 100-level nested structure+ nested: dict = {"leaf": "value"}+ for _i in range(100):+ nested = {"level": nested}++ content = json.dumps(nested)+ result, was_modified, info = crusher._smart_crush_content(content)+ # Should not raise RecursionError+ parsed = json.loads(result)+ # The deep structure should be preserved (returned as-is past depth limit)+ assert isinstance(parsed, dict)++ def test_deeply_nested_list_does_not_crash(self) -> None:+ """Deeply nested lists should also be handled safely."""+ crusher = _make_crusher()+ nested: list = ["leaf"]+ for _i in range(100):+ nested = [nested]++ content = json.dumps(nested)+ result, was_modified, info = crusher._smart_crush_content(content)+ parsed = json.loads(result)+ assert isinstance(parsed, list)