Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(rust): retire python smart_crusher, ship rust-only via pyo3
chopratejasc765c53bf851
diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.pyindex db8d284e8..dff901a43 100644--- a/headroom/transforms/smart_crusher.py+++ b/headroom/transforms/smart_crusher.py@@ -1,3717 +1,329 @@-"""Smart statistical tool output compression for Headroom SDK.--This module provides intelligent JSON compression based on statistical analysis-rather than fixed rules. It analyzes data patterns and applies optimal compression-strategies to maximize token reduction while preserving important information.--SCOPE: SmartCrusher handles JSON arrays of ANY type — dicts, strings, numbers,-mixed types, and nested arrays. Non-JSON content (plain text, search results,-logs, code, diffs) passes through UNCHANGED.--TEXT COMPRESSION IS OPT-IN: For text-based content, Headroom provides standalone-utilities that applications can use explicitly:-- SearchCompressor: For grep/ripgrep output (file:line:content format)-- LogCompressor: For build/test logs (pytest, npm, cargo output)-- Kompress: For generic plain text (ML-based, requires [ml] extra)--Applications should decide when and how to use text compression based on their-specific needs. This design prevents lossy text compression from being applied-automatically, which could lose important context in coding tasks.--SCHEMA-PRESERVING: Output contains only items from the original array.-No wrappers, no generated text, no metadata keys. This ensures downstream-tools and parsers work unchanged.--Supported JSON types:-- Arrays of dicts: Full statistical analysis with adaptive K (Kneedle algorithm)-- Arrays of strings: Dedup + adaptive sampling + error preservation-- Arrays of numbers: Statistical summary + outlier/change-point preservation-- Mixed-type arrays: Grouped by type, each group compressed independently-- Flat objects (many keys): Key-level adaptive sampling-- Nested objects: Recursive compression of inner arrays/objects--Safety guarantees (consistent across ALL types):-- First K, last K items always kept (K is adaptive, not hardcoded)-- Error items (containing 'error', 'exception', 'failed', 'critical') never dropped-- Anomalous numeric items (> 2 std from mean) always kept-- Items around detected change points preserved-- Items with high relevance score to user query (via RelevanceScorer)--Key Features:-- RelevanceScorer: ML-powered or BM25-based relevance matching (replaces regex)-- Variance-based change point detection (preserve anomalies)-- Error item detection (never lose error messages)-- Pattern detection (time series, logs, search results)-- Strategy selection based on data characteristics-"""--from __future__ import annotations--import hashlib-import json-import logging-import math-import re-import statistics-import threading-from collections import Counter-from dataclasses import dataclass, field-from enum import Enum-from typing import Any--from ..cache.compression_feedback import CompressionFeedback, get_compression_feedback-from ..cache.compression_store import CompressionStore, get_compression_store-from ..config import AnchorConfig, CCRConfig, RelevanceScorerConfig, TransformResult-from ..relevance import RelevanceScorer, create_scorer-from ..telemetry import TelemetryCollector, ToolSignature, get_telemetry_collector-from ..telemetry.models import FieldSemantics-from ..telemetry.toin import ToolIntelligenceNetwork, get_toin-from ..tokenizer import Tokenizer-from ..utils import (- compute_short_hash,- create_tool_digest_marker,- deep_copy_messages,- safe_json_dumps,- safe_json_loads,-)-from .anchor_selector import AnchorSelector-from .anchor_selector import DataPattern as AnchorDataPattern-from .base import Transform-from .error_detection import ERROR_KEYWORDS--logger = logging.getLogger(__name__)--# Legacy patterns for backwards compatibility (extract_query_anchors)-_UUID_PATTERN = re.compile(- r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"-)-_NUMERIC_ID_PATTERN = re.compile(r"\b\d{4,}\b") # 4+ digit numbers (likely IDs)-_HOSTNAME_PATTERN = re.compile(- r"\b[a-zA-Z0-9][-a-zA-Z0-9]*\.[a-zA-Z0-9][-a-zA-Z0-9]*(?:\.[a-zA-Z]{2,})?\b"-)-_QUOTED_STRING_PATTERN = re.compile(r"['\"]([^'\"]{1,50})['\"]") # Short quoted strings-_EMAIL_PATTERN = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b")--# Temporal detection patterns (compiled once, used in SmartAnalyzer._detect_temporal_field)-_ISO_DATETIME_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}")-_ISO_DATE_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}$")---def extract_query_anchors(text: str) -> set[str]:- """Extract query anchors from user text (legacy regex-based method).-- DEPRECATED: Use RelevanceScorer.score_batch() for better semantic matching.-- Query anchors are identifiers or values that the user is likely searching for.- When crushing tool outputs, items matching these anchors should be preserved.-- Extracts:- - UUIDs (e.g., "550e8400-e29b-41d4-a716-446655440000")- - Numeric IDs (4+ digits, e.g., "12345", "1001234")- - Hostnames (e.g., "api.example.com", "server-01.prod")- - Quoted strings (e.g., 'Alice', "error_code")- - Email addresses (e.g., "[email protected]")-- Args:- text: User message text to extract anchors from.-- Returns:- Set of anchor strings (lowercased for case-insensitive matching).- """- anchors: set[str] = set()-- if not text:- return anchors-- # UUIDs- for match in _UUID_PATTERN.findall(text):- anchors.add(match.lower())-- # Numeric IDs- for match in _NUMERIC_ID_PATTERN.findall(text):- anchors.add(match)-- # Hostnames- for match in _HOSTNAME_PATTERN.findall(text):- # Filter out common false positives- if match.lower() not in ("e.g", "i.e", "etc."):- anchors.add(match.lower())-- # Quoted strings- for match in _QUOTED_STRING_PATTERN.findall(text):- if len(match.strip()) >= 2: # Skip very short matches- anchors.add(match.lower())-- # Email addresses- for match in _EMAIL_PATTERN.findall(text):- anchors.add(match.lower())-- return anchors---def item_matches_anchors(item: dict, anchors: set[str]) -> bool:- """Check if an item matches any query anchors (legacy method).-- DEPRECATED: Use RelevanceScorer for better matching.-- Args:- item: Dictionary item from tool output.- anchors: Set of anchor strings to match.-- Returns:- True if any anchor is found in the item's string representation.- """- if not anchors:- return False-- item_str = str(item).lower()- return any(anchor in item_str for anchor in anchors)---def _hash_field_name(field_name: str) -> str:- """Hash a field name to match TOIN's anonymized preserve_fields.-- TOIN stores field names as SHA256[:8] hashes for privacy.- This function produces the same hash format.- """- return hashlib.sha256(field_name.encode()).hexdigest()[:8]---# Minimum chars for a text field to be worth compressing within an item-_MIN_FIELD_CHARS_FOR_WITHIN = 200--# Lazy-loaded compressor for within-item text compression (thread-safe)-_within_compressor: Any = None-_within_compressor_checked = False-_within_compressor_lock = threading.Lock()---def _get_within_compressor() -> Any:- """Get a text compressor for within-item field compression.-- Returns Kompress if available (requires [ml] extra), else None.- Thread-safe via double-checked locking.- """- global _within_compressor, _within_compressor_checked- if not _within_compressor_checked:- with _within_compressor_lock:- if not _within_compressor_checked:- try:- from .kompress_compressor import KompressCompressor, is_kompress_available-- if is_kompress_available():- _within_compressor = KompressCompressor()- logger.debug("Within-item compression: using Kompress")- except ImportError:- pass- _within_compressor_checked = True- return _within_compressor---def _compress_text_within_items(items: list[dict], context: str = "") -> list[dict]:- """Compress long text fields WITHIN each item, keeping all items.-- Used when diversity is high (all items are unique) — instead of dropping- items, compress the verbose text inside each one. Falls back to the- original list unchanged if no compressor is available or no field is- long enough to benefit.-- Args:- items: JSON-parsed list of dicts.- context: User query context for relevance-aware compression.-- Returns:- Compressed items (new list) or the *same* ``items`` object if- nothing was compressed (caller checks identity).- """- compressor = _get_within_compressor()- if compressor is None:- return items # No ML compressor available — pass through-- any_compressed = False- result: list[dict] = []-- for item in items:- new_item = dict(item) # Shallow copy- item_changed = False-- for key, value in item.items():- if not isinstance(value, str) or len(value) < _MIN_FIELD_CHARS_FOR_WITHIN:- continue-- try:- compressed = compressor.compress(value, context=context)- if compressed.compressed and len(compressed.compressed) < len(value) * 0.9:- new_item[key] = compressed.compressed- item_changed = True- except Exception:- pass # Compression failed for this field — keep original-- result.append(new_item if item_changed else item)- if item_changed:- any_compressed = True-- return result if any_compressed else items---def _get_preserve_field_values(- item: dict,- preserve_field_hashes: list[str],-) -> list[tuple[str, Any]]:- """Get values from item fields that match TOIN's preserve_field hashes.-- TOIN stores preserve_fields as hashed field names (SHA256[:8]).- This function iterates over item fields, hashes each, and returns- matching field names and values.-- Args:- item: Dictionary item from tool output.- preserve_field_hashes: List of SHA256[:8] hashed field names from TOIN.-- Returns:- List of (field_name, value) tuples for fields that match.- """- if not preserve_field_hashes or not item:- return []-- # Convert preserve_fields to set for O(1) lookup- hash_set = set(preserve_field_hashes)-- matches = []- for field_name, value in item.items():- field_hash = _hash_field_name(field_name)- if field_hash in hash_set:- matches.append((field_name, value))-- return matches---def _percentile_linear(sorted_values: list[float], q: float) -> float:- """Linear-interpolation percentile, matching numpy's "linear" method.-- BUG #1 FIX (replaces integer-division indexing in _crush_number_array- that was off by one for `len < 8`). Computes:- index = q * (n - 1)- if index is integer: return sorted_values[index]- else: linear interpolate between floor and ceil-- Args:- sorted_values: Pre-sorted ascending list of finite numbers.- q: Quantile in [0, 1] (e.g. 0.25 for p25, 0.75 for p75).-- Returns:- The interpolated quantile value. Empty input returns 0.0.- """- n = len(sorted_values)- if n == 0:- return 0.0- if n == 1:- return float(sorted_values[0])- pos = q * (n - 1)- lo = int(pos)- hi = lo + 1 if lo + 1 < n else lo- frac = pos - lo- return sorted_values[lo] * (1 - frac) + sorted_values[hi] * frac---def _item_has_preserve_field_match(- item: dict,- preserve_field_hashes: list[str],- query_context: str,-) -> bool:- """Check if item has a preserve_field value that matches query context.-- Args:- item: Dictionary item from tool output.- preserve_field_hashes: List of SHA256[:8] hashed field names from TOIN.- query_context: User's query to match against field values.-- Returns:- True if any preserve_field value matches the query context.- """- if not query_context:- return False-- query_lower = query_context.lower()-- for _field_name, value in _get_preserve_field_values(item, preserve_field_hashes):- if value is not None:- value_str = str(value).lower()- if value_str in query_lower or query_lower in value_str:- return True-- return False---class CompressionStrategy(Enum):- """Compression strategies based on data patterns."""-- NONE = "none" # No compression needed- SKIP = "skip" # Explicitly skip - not safe to crush- TIME_SERIES = "time_series" # Keep change points, summarize stable- CLUSTER_SAMPLE = "cluster" # Dedupe similar items- TOP_N = "top_n" # Keep highest scored items- SMART_SAMPLE = "smart_sample" # Statistical sampling with constants---class ArrayType(Enum):- """JSON array element type classification."""-- DICT_ARRAY = "dict_array" # [{...}, {...}, ...]- STRING_ARRAY = "string_array" # ["a", "b", "c", ...]- NUMBER_ARRAY = "number_array" # [1, 2.5, 3, ...]- BOOL_ARRAY = "bool_array" # [true, false, ...]- NESTED_ARRAY = "nested_array" # [[...], [...], ...]- MIXED_ARRAY = "mixed_array" # [{"a":1}, "str", 42, ...]- EMPTY = "empty"---def _classify_array(items: list) -> ArrayType:- """Classify a JSON array by its element types.-- Uses set-of-types check on ALL elements (not sampling) to guarantee- correct classification. Fast because type() is O(1).- """- if not items:- return ArrayType.EMPTY- # Note: bool is a subclass of int in Python, so check bool first- types = set()- has_bool = False- for item in items:- if isinstance(item, bool):- has_bool = True- types.add(type(item))- if has_bool and types <= {bool, int}:- # All bools (Python's True/False are int subclass)- if all(isinstance(i, bool) for i in items):- return ArrayType.BOOL_ARRAY- if types == {dict}:- return ArrayType.DICT_ARRAY- if types == {str}:- return ArrayType.STRING_ARRAY- if types <= {int, float} and not has_bool:- return ArrayType.NUMBER_ARRAY- if types == {list}:- return ArrayType.NESTED_ARRAY- return ArrayType.MIXED_ARRAY---# =====================================================================-# STATISTICAL FIELD DETECTION (replaces hardcoded string patterns)-# =====================================================================-# Instead of matching field names like "id", "score", "error", we use-# statistical and structural properties of the data to detect field types.---def _is_uuid_format(value: str) -> bool:- """Check if a string looks like a UUID (structural pattern)."""- if not isinstance(value, str) or len(value) != 36:- return False- # UUID format: 8-4-4-4-12 hex chars- parts = value.split("-")- if len(parts) != 5:- return False- expected_lens = [8, 4, 4, 4, 12]- for part, expected_len in zip(parts, expected_lens):- if len(part) != expected_len:- return False- if not all(c in "0123456789abcdefABCDEF" for c in part):- return False- return True---def _calculate_string_entropy(s: str) -> float:- """Calculate Shannon entropy of a string, normalized to [0, 1].-- High entropy (>0.7) suggests random/ID-like content.- Low entropy (<0.3) suggests repetitive/predictable content.- """- if not s or len(s) < 2:- return 0.0-- # Count character frequencies- freq: dict[str, int] = {}- for c in s:- freq[c] = freq.get(c, 0) + 1-- # Calculate entropy- import math-- entropy = 0.0- length = len(s)- for count in freq.values():- p = count / length- if p > 0:- entropy -= p * math.log2(p)-- # Normalize by max possible entropy for this length- max_entropy = math.log2(min(len(freq), length))- if max_entropy > 0:- return entropy / max_entropy- return 0.0---def _detect_sequential_pattern(values: list[Any], check_order: bool = True) -> bool:- """Detect if numeric values form a sequential pattern (like IDs: 1,2,3,...).-- Returns True if values appear to be auto-incrementing or sequential.-- Args:- values: List of values to check.- check_order: If True, also check if values are in ascending order in the array.- Score fields are often sorted descending, while IDs are ascending.- """- if len(values) < 5:- return False-- # BUG #2 FIX: track whether ANY value was already numeric (not- # just a stringified number). If every "number" came from `int(s)`- # of a string value, the field is almost certainly a categorical- # string ID — possibly zero-padded — and should NOT be flagged- # as sequential. Without this guard, `["001", "002", "003"]`- # silently becomes `[1, 2, 3]` and gets misclassified.- nums = []- had_non_string_numeric = False- for v in values:- if isinstance(v, int | float) and not isinstance(v, bool):- nums.append(v)- had_non_string_numeric = True- elif isinstance(v, str):- try:- nums.append(int(v))- # Intentionally do NOT set had_non_string_numeric —- # see fix doc above.- except ValueError:- pass-- if len(nums) < 5:- return False-- # BUG #2 FIX: if every parseable value came from a string, the- # field is categorical (zero-padded codes, alphanumeric IDs that- # happen to be int-parseable, etc.). Don't classify as sequential.- if not had_non_string_numeric:- return False-- # Need at least 2 elements for pairwise comparison- if len(nums) < 2:- return False-- # Check if sorted values form a near-sequence- sorted_nums = sorted(nums)- diffs = [sorted_nums[i + 1] - sorted_nums[i] for i in range(len(sorted_nums) - 1)]-- if not diffs:- return False-- # If most differences are 1 (or small constant), it's sequential- avg_diff = sum(diffs) / len(diffs)- if 0.5 <= avg_diff <= 2.0:- # Check consistency - sequential IDs have consistent spacing- consistent_count = sum(1 for d in diffs if 0.5 <= d <= 2.0)- is_sequential = consistent_count / len(diffs) > 0.8-- # Additional check: IDs are typically in ASCENDING order in the array- # Scores sorted by relevance are typically in DESCENDING order- if check_order and is_sequential:- # Check if original order is ascending (like IDs)- ascending_count = sum(1 for i in range(len(nums) - 1) if nums[i] <= nums[i + 1])- is_ascending = ascending_count / (len(nums) - 1) > 0.7- return is_ascending # Only flag as sequential if ascending (ID-like)-- return is_sequential-- return False---def _detect_id_field_statistically(stats: FieldStats, values: list[Any]) -> tuple[bool, float]:- """Detect if a field is an ID field using statistical properties.-- Returns (is_id_field, confidence).-- ID fields have:- - Very high uniqueness (>0.95)- - Sequential numeric pattern OR UUID format OR high entropy strings- """- # Must have high uniqueness- if stats.unique_ratio < 0.9:- return False, 0.0-- confidence = 0.0-- # Check for UUID format (structural detection)- if stats.field_type == "string":- sample_values = [v for v in values[:20] if isinstance(v, str)]- uuid_count = sum(1 for v in sample_values if _is_uuid_format(v))- if sample_values and uuid_count / len(sample_values) > 0.8:- return True, 0.95-- # Check for high entropy (random string IDs)- if sample_values:- avg_entropy = sum(_calculate_string_entropy(v) for v in sample_values) / len(- sample_values- )- if avg_entropy > 0.7 and stats.unique_ratio > 0.95:- confidence = 0.8- return True, confidence-- # Check for sequential numeric pattern- if stats.field_type == "numeric":- if _detect_sequential_pattern(values) and stats.unique_ratio > 0.95:- return True, 0.9-- # High uniqueness numeric with high range suggests ID- if stats.min_val is not None and stats.max_val is not None:- value_range = stats.max_val - stats.min_val- if value_range > 0 and stats.unique_ratio > 0.95:- return True, 0.85-- # Very high uniqueness alone is a signal (even without other patterns)- if stats.unique_ratio > 0.98:- return True, 0.7-- return False, 0.0---def _detect_score_field_statistically(stats: FieldStats, items: list[dict]) -> tuple[bool, float]:- """Detect if a field is a score/ranking field using statistical properties.-- Returns (is_score_field, confidence).-- Score fields have:- - Numeric type- - Bounded range (0-1, 0-10, 0-100, or similar)- - NOT sequential (unlike IDs)- - Often the data appears sorted by this field (descending)- """- if stats.field_type != "numeric":- return False, 0.0-- if stats.min_val is None or stats.max_val is None:- return False, 0.0-- confidence = 0.0-- # Check for bounded range typical of scores- min_val, max_val = stats.min_val, stats.max_val-- # Common score ranges: [0,1], [0,10], [0,100], [-1,1], [0,5]- is_bounded = False- if 0 <= min_val <= 1 and 0 <= max_val <= 1: # [0,1] range- is_bounded = True- confidence += 0.4- elif 0 <= min_val <= 10 and 0 <= max_val <= 10: # [0,10] range- is_bounded = True- confidence += 0.3- elif 0 <= min_val <= 100 and 0 <= max_val <= 100: # [0,100] range- is_bounded = True- confidence += 0.25- elif -1 <= min_val and max_val <= 1: # [-1,1] range- is_bounded = True- confidence += 0.35-- if not is_bounded:- return False, 0.0-- # Should NOT be sequential (IDs are sequential, scores are not)- sample_values = [item.get(stats.name) for item in items[:50] if stats.name in item]- if _detect_sequential_pattern(sample_values):- return False, 0.0-- # Check if data appears sorted by this field (descending = relevance sorted)- # Filter out NaN/Inf which break comparisons- values_in_order: list[float] = []- for item in items:- if stats.name in item:- val = item.get(stats.name)- if isinstance(val, int | float) and math.isfinite(val):- values_in_order.append(float(val))- if len(values_in_order) >= 5:- # Check for descending sort- num_pairs = len(values_in_order) - 1- descending_count = sum(- 1 for i in range(num_pairs) if values_in_order[i] >= values_in_order[i + 1]- )- if num_pairs > 0 and descending_count / num_pairs > 0.7:- confidence += 0.3-- # Score fields often have floating point values- # Filter out NaN/Inf which can't be converted to int- float_count = sum(- 1 for v in values_in_order[:20] if isinstance(v, float) and math.isfinite(v) and v != int(v)- )- if float_count > len(values_in_order[:20]) * 0.3:- confidence += 0.1-- return confidence >= 0.4, min(confidence, 0.95)---def _detect_structural_outliers(items: list[dict]) -> list[int]:- """Detect items that are structural outliers (error-like items).-- Instead of looking for "error" keywords, we detect:- 1. Items with extra fields that others don't have- 2. Items with rare status/state values- 3. Items with significantly different structure-- Returns indices of outlier items.- """- if len(items) < 5:- return []-- outlier_indices: list[int] = []-- # 1. Detect items with extra fields- # Find the "common" field set (fields present in >80% of items)- field_counts: dict[str, int] = {}- for item in items:- if isinstance(item, dict):- for key in item.keys():- field_counts[key] = field_counts.get(key, 0) + 1-- n = len(items)- common_fields = {k for k, v in field_counts.items() if v >= n * 0.8}- rare_fields = {k for k, v in field_counts.items() if v < n * 0.2}-- for i, item in enumerate(items):- if not isinstance(item, dict):- continue-- item_fields = set(item.keys())-- # Has rare fields that most items don't have- has_rare = bool(item_fields & rare_fields)- if has_rare:- outlier_indices.append(i)- continue-- # 2. Detect rare status/state values- # Find fields that look like status fields (low cardinality, categorical)- status_outliers = _detect_rare_status_values(items, common_fields)- outlier_indices.extend(status_outliers)-- return list(set(outlier_indices))---def _detect_rare_status_values(items: list[dict], common_fields: set[str]) -> list[int]:- """Detect items with rare values in status-like fields.-- A status field has low cardinality (few distinct values).- If 95%+ have the same value, items with different values are interesting.- """- outlier_indices: list[int] = []-- # Find potential status fields (low cardinality)- for field_name in common_fields:- values = [- item.get(field_name) for item in items if isinstance(item, dict) and field_name in item- ]-- # Skip if too few values or non-hashable- try:- unique_values = {str(v) for v in values if v is not None}- except Exception:- continue-- # BUG #3 FIX: cardinality cap raised from 10 to 50 so that- # higher-cardinality but Pareto-distributed fields (e.g.- # 60 INFO + 25 WARN + 15 distinct error codes) still get- # their rare values flagged. Above 50 distinct values, the- # field is almost certainly an ID/free-form column, not a- # status enum.- if not (2 <= len(unique_values) <= 50):- continue-- # Count value frequencies- value_counts: dict[str, int] = {}- for v in values:- key = str(v) if v is not None else "__none__"- value_counts[key] = value_counts.get(key, 0) + 1-- if not value_counts:- continue-- total = len(values)-- # BUG #3 FIX: replace single-dominant check with Pareto top-K.- # Sort frequencies descending; tiebreak by key ascending so the- # outcome is deterministic when multiple values have the same- # frequency. Find the smallest K such that top-K covers >=80%- # of items.- sorted_counts = sorted(value_counts.items(), key=lambda x: (-x[1], x[0]))- threshold = math.ceil(total * 0.8)- cumulative = 0- top_k_values: set[str] = set()- for value, count in sorted_counts:- cumulative += count- top_k_values.add(value)- if cumulative >= threshold:- break-- # K must be small (<=5) for any value to count as "rare".- # Above this the distribution is too uniform to label a value- # rare.- if len(top_k_values) > 5:- continue-- # Items NOT in top_k_values are outliers.- for i, item in enumerate(items):- if not isinstance(item, dict) or field_name not in item:- continue- item_value = str(item[field_name]) if item[field_name] is not None else "__none__"- if item_value not in top_k_values:- outlier_indices.append(i)-- return outlier_indices---# Error keywords for PRESERVATION guarantee (not crushability detection)-# This is for the quality guarantee: "ALL error items are ALWAYS preserved"-# regardless of how common they are. Used in _prioritize_indices().-# Centralized in error_detection module for consistency across transforms.-_ERROR_KEYWORDS_FOR_PRESERVATION = ERROR_KEYWORDS---def _detect_error_items_for_preservation(- items: list[dict],- item_strings: list[str] | None = None,-) -> list[int]:- """Detect items containing error keywords for PRESERVATION guarantee.-- This is NOT for crushability analysis - it's for ensuring ALL error items- are retained during compression. The quality guarantee is that error items- are NEVER dropped, even if errors are common in the dataset.-- Uses keywords because error semantics are well-defined across domains.-- Args:- items: List of items to check.- item_strings: Pre-computed JSON serializations to avoid redundant json.dumps.- """- error_indices: list[int] = []-- for i, item in enumerate(items):- if not isinstance(item, dict):- continue-- # Reuse cached serialization if available, otherwise serialize- try:- if item_strings is not None and i < len(item_strings):- item_str = item_strings[i].lower()- else:- item_str = json.dumps(item).lower()- except Exception:- continue-- # Check if any error keyword is present- for keyword in _ERROR_KEYWORDS_FOR_PRESERVATION:- if keyword in item_str:- error_indices.append(i)- break-- return error_indices---def _detect_items_by_learned_semantics(- items: list[dict],- field_semantics: dict[str, FieldSemantics],-) -> list[int]:- """Detect items with important values based on learned field semantics.-- This is the TOIN Evolution integration - uses learned field semantic types- to identify items that should be preserved during compression.-- Key insight: Instead of hardcoded patterns, we learn from user behavior- which field values are actually important (e.g., error indicators, rare- status values, identifiers that get queried).-- Args:- items: List of items to analyze.- field_semantics: Learned field semantics from TOIN (field_hash -> FieldSemantics).-- Returns:- List of indices for items containing important values.- """- if not field_semantics or not items:- return []-- important_indices: list[int] = []-- # Build a quick lookup for field_hash -> FieldSemantics- # Pre-filter to fields with sufficient confidence- confident_semantics = {- fh: fs- for fh, fs in field_semantics.items()- if fs.confidence >= 0.3 and fs.inferred_type != "unknown"- }-- if not confident_semantics:- return []-- # Pre-compute field name hashes to avoid redundant SHA256 per item- _field_hash_cache: dict[str, str] = {}-- for i, item in enumerate(items):- if not isinstance(item, dict):- continue-- for field_name, value in item.items():- # Hash the field name to match TOIN's format (cached per unique field name)- if field_name not in _field_hash_cache:- _field_hash_cache[field_name] = _hash_field_name(field_name)- field_hash = _field_hash_cache[field_name]-- if field_hash not in confident_semantics:- continue-- field_sem = confident_semantics[field_hash]-- # Hash the value to check importance- if value is None:- value_canonical = "null"- elif isinstance(value, bool):- value_canonical = "true" if value else "false"- elif isinstance(value, int | float):- value_canonical = str(value)- elif isinstance(value, str):- value_canonical = value- elif isinstance(value, list | dict):- try:- value_canonical = json.dumps(value, sort_keys=True, default=str)- except (TypeError, ValueError):- value_canonical = str(value)- else:- value_canonical = str(value)-- value_hash = hashlib.sha256(value_canonical.encode()).hexdigest()[:8]-- # Check if this value is important based on learned semantics- if field_sem.is_value_important(value_hash):- important_indices.append(i)- break # Only need to mark item once-- return important_indices---@dataclass-class CrushabilityAnalysis:- """Analysis of whether an array is safe to crush.-- The key insight: if we don't have a reliable SIGNAL to determine- which items are important, we should NOT crush at all.-- Signals include:- - Score/rank fields (search results)- - Error keywords (logs)- - Numeric anomalies (metrics)- - Low uniqueness (repetitive data where sampling is representative)-- High variability + No signal = DON'T CRUSH- """-- crushable: bool- confidence: float # 0.0 to 1.0- reason: str- signals_present: list[str] = field(default_factory=list)- signals_absent: list[str] = field(default_factory=list)-- # Detailed metrics- has_id_field: bool = False- id_uniqueness: float = 0.0- avg_string_uniqueness: float = 0.0- has_score_field: bool = False- error_item_count: int = 0- anomaly_count: int = 0---@dataclass-class FieldStats:- """Statistics for a single field across array items."""-- name: str- field_type: str # "numeric", "string", "boolean", "object", "array", "null"- count: int- unique_count: int- unique_ratio: float- is_constant: bool- constant_value: Any = None-- # Numeric-specific stats- min_val: float | None = None- max_val: float | None = None- mean_val: float | None = None- variance: float | None = None- change_points: list[int] = field(default_factory=list)-- # String-specific stats- avg_length: float | None = None- top_values: list[tuple[str, int]] = field(default_factory=list)---@dataclass-class ArrayAnalysis:- """Complete analysis of an array."""-- item_count: int- field_stats: dict[str, FieldStats]- detected_pattern: str # "time_series", "logs", "search_results", "generic"- recommended_strategy: CompressionStrategy- constant_fields: dict[str, Any]- estimated_reduction: float- crushability: CrushabilityAnalysis | None = None # Whether it's safe to crush---@dataclass-class CompressionPlan:- """Plan for how to compress an array."""-- strategy: CompressionStrategy- keep_indices: list[int] = field(default_factory=list)- constant_fields: dict[str, Any] = field(default_factory=dict)- summary_ranges: list[tuple[int, int, dict]] = field(default_factory=list)- cluster_field: str | None = None- sort_field: str | None = None- keep_count: int = 10---@dataclass-class CrushResult:- """Result from SmartCrusher.crush() method.-- Used by ContentRouter when routing JSON arrays to SmartCrusher.- """-- compressed: str- original: str- was_modified: bool- strategy: str = "passthrough"---@dataclass-class SmartCrusherConfig:- """Configuration for smart crusher.-- SCHEMA-PRESERVING: Output contains only items from the original array.- No wrappers, no generated text, no metadata keys.- """-- enabled: bool = True- min_items_to_analyze: int = 5 # Don't analyze tiny arrays- min_tokens_to_crush: int = 200 # Only crush if > N tokens- variance_threshold: float = 2.0 # Std devs for change point detection- uniqueness_threshold: float = 0.1 # Below this = nearly constant- similarity_threshold: float = 0.8 # For clustering similar strings- max_items_after_crush: int = 15 # Target max items in output- preserve_change_points: bool = True- factor_out_constants: bool = False # Disabled - preserves original schema- include_summaries: bool = False # Disabled - no generated text-- # Feedback loop integration- use_feedback_hints: bool = True # Use learned patterns to adjust compression-- # LOW FIX #21: Make TOIN confidence threshold configurable- # Minimum confidence required to apply TOIN recommendations- toin_confidence_threshold: float = 0.5-- # Content deduplication - prevents wasting slots on identical items- dedup_identical_items: bool = True-- # Adaptive K boundary allocation (fraction of total K for first/last items)- first_fraction: float = 0.3 # 30% of K from start of array- last_fraction: float = 0.15 # 15% of K from end of array---class SmartAnalyzer:- """Analyzes JSON arrays to determine optimal compression strategy."""-- def __init__(self, config: SmartCrusherConfig | None = None):- self.config = config or SmartCrusherConfig()-- def analyze_array(self, items: list[dict]) -> ArrayAnalysis:- """Perform complete statistical analysis of an array."""- if not items or not isinstance(items[0], dict):- return ArrayAnalysis(- item_count=len(items) if items else 0,- field_stats={},- detected_pattern="generic",- recommended_strategy=CompressionStrategy.NONE,- constant_fields={},- estimated_reduction=0.0,- )-- # Analyze each field- field_stats = {}- all_keys: set[str] = set()- for item in items:- if isinstance(item, dict):- all_keys.update(item.keys())-- # PARITY FIX (Stage 3c.1): iterate in sorted key order. Python's- # set iteration is non-deterministic across PYTHONHASHSEED, so- # downstream short-circuits (`_select_strategy` "message" lookup,- # `_detect_pattern` first-score-field) become non-deterministic.- # Rust uses BTreeMap which iterates ASCII-sorted; sorting here- # locks both languages to the same iteration order so parity- # fixtures byte-match.- for key in sorted(all_keys):- field_stats[key] = self._analyze_field(key, items)-- # Detect pattern- pattern = self._detect_pattern(field_stats, items)-- # Extract constants- constant_fields = {k: v.constant_value for k, v in field_stats.items() if v.is_constant}-- # CRITICAL: Analyze crushability BEFORE selecting strategy- crushability = self.analyze_crushability(items, field_stats)-- # Select strategy (respects crushability)- strategy = self._select_strategy(field_stats, pattern, len(items), crushability)-- # Estimate reduction (0 if not crushable)- if strategy == CompressionStrategy.SKIP:- reduction = 0.0- else:- reduction = self._estimate_reduction(field_stats, strategy, len(items))-- return ArrayAnalysis(- item_count=len(items),- field_stats=field_stats,- detected_pattern=pattern,- recommended_strategy=strategy,- constant_fields=constant_fields,- estimated_reduction=reduction,- crushability=crushability,- )-- def _analyze_field(self, key: str, items: list[dict]) -> FieldStats:- """Analyze a single field across all items."""- values = [item.get(key) for item in items if isinstance(item, dict)]- non_null_values = [v for v in values if v is not None]-- if not non_null_values:- return FieldStats(- name=key,- field_type="null",- count=len(values),- unique_count=0,- unique_ratio=0.0,- is_constant=True,- constant_value=None,- )-- # Determine type from first non-null value- first_val = non_null_values[0]- if isinstance(first_val, bool):- field_type = "boolean"- elif isinstance(first_val, int | float):- field_type = "numeric"- elif isinstance(first_val, str):- field_type = "string"- elif isinstance(first_val, dict):- field_type = "object"- elif isinstance(first_val, list):- field_type = "array"- else:- field_type = "unknown"-- # Compute uniqueness- str_values = [str(v) for v in values]- unique_values = set(str_values)- unique_count = len(unique_values)- unique_ratio = unique_count / len(values) if values else 0-- # Check if constant- is_constant = unique_count == 1- constant_value = non_null_values[0] if is_constant else None-- stats = FieldStats(- name=key,- field_type=field_type,- count=len(values),- unique_count=unique_count,- unique_ratio=unique_ratio,- is_constant=is_constant,- constant_value=constant_value,- )-- # Numeric-specific analysis- if field_type == "numeric":- # Filter out NaN and Infinity which break statistics functions- nums = [v for v in non_null_values if isinstance(v, int | float) and math.isfinite(v)]- if nums:- try:- stats.min_val = min(nums)- stats.max_val = max(nums)- stats.mean_val = statistics.mean(nums)- stats.variance = statistics.variance(nums) if len(nums) > 1 else 0- stats.change_points = self._detect_change_points(nums)- except (OverflowError, ValueError):- # Extreme values that overflow - skip detailed statistics- stats.min_val = None- stats.max_val = None- stats.mean_val = None- stats.variance = 0- stats.change_points = []-- # String-specific analysis- elif field_type == "string":- strs = [v for v in non_null_values if isinstance(v, str)]- if strs:- stats.avg_length = statistics.mean(len(s) for s in strs)- stats.top_values = Counter(strs).most_common(5)-- return stats-- def _detect_change_points(self, values: list[float], window: int = 5) -> list[int]:- """Detect indices where values change significantly."""- if len(values) < window * 2:- return []-- change_points = []-- # Calculate overall statistics- overall_std = statistics.stdev(values) if len(values) > 1 else 0- if overall_std == 0:- return []-- threshold = self.config.variance_threshold * overall_std-- # Sliding window comparison- for i in range(window, len(values) - window):- before_mean = statistics.mean(values[i - window : i])- after_mean = statistics.mean(values[i : i + window])-- if abs(after_mean - before_mean) > threshold:- change_points.append(i)-- # Deduplicate nearby change points- if change_points:- deduped = [change_points[0]]- for cp in change_points[1:]:- if cp - deduped[-1] > window:- deduped.append(cp)- return deduped-- return []-- def _detect_pattern(self, field_stats: dict[str, FieldStats], items: list[dict]) -> str:- """Detect the data pattern using STATISTICAL analysis (no hardcoded field names).-- Pattern detection:- - TIME_SERIES: Has a temporal field (detected by value format) + numeric variance- - LOGS: Has a high-cardinality string field + low-cardinality categorical field- - SEARCH_RESULTS: Has a score-like field (bounded numeric, possibly sorted)- - GENERIC: Default- """- # Check for time series pattern using STRUCTURAL detection- has_timestamp = self._detect_temporal_field(field_stats, items)-- numeric_fields = [k for k, v in field_stats.items() if v.field_type == "numeric"]- has_numeric_with_variance = any(- (field_stats[k].variance is not None and (field_stats[k].variance or 0) > 0)- for k in numeric_fields- )-- if has_timestamp and has_numeric_with_variance:- return "time_series"-- # Check for logs pattern using STATISTICAL detection- # Logs have: high-cardinality string (message) + low-cardinality categorical (level)- has_message_like = False- has_level_like = False-- for _name, stats in field_stats.items():- if stats.field_type == "string":- # High-cardinality string = likely message field- if stats.unique_ratio > 0.5 and stats.avg_length and stats.avg_length > 20:- has_message_like = True- # Low-cardinality string = likely level/status field- elif stats.unique_ratio < 0.1 and 2 <= stats.unique_count <= 10:- has_level_like = True-- if has_message_like and has_level_like:- return "logs"-- # Check for search results pattern using STATISTICAL score detection- for _name, stats in field_stats.items():- is_score, confidence = _detect_score_field_statistically(stats, items)- if is_score and confidence >= 0.5:- return "search_results"-- return "generic"-- def _detect_temporal_field(self, field_stats: dict[str, FieldStats], items: list[dict]) -> bool:- """Detect if any field contains temporal values (dates/timestamps).-- Uses STRUCTURAL detection based on value format, not field names.- """- # Check string fields for ISO 8601 patterns (module-level compiled)- iso_datetime_pattern = _ISO_DATETIME_PATTERN- iso_date_pattern = _ISO_DATE_PATTERN-- for name, stats in field_stats.items():- if stats.field_type == "string":- # Sample some values- sample_values = [- item.get(name) for item in items[:10] if isinstance(item.get(name), str)- ]- if sample_values:- # Check if values look like dates/datetimes- iso_count = sum(- 1- for v in sample_values- if v is not None- and (iso_datetime_pattern.match(v) or iso_date_pattern.match(v))- )- if iso_count / len(sample_values) > 0.5:- return True-- # Check numeric fields for Unix timestamp range- elif stats.field_type == "numeric":- if stats.min_val and stats.max_val:- # Unix timestamps (seconds): 1000000000 to 2000000000 (roughly 2001-2033)- # Unix timestamps (milliseconds): 1000000000000 to 2000000000000- is_unix_seconds = 1000000000 <= stats.min_val <= 2000000000- is_unix_millis = 1000000000000 <= stats.min_val <= 2000000000000- if is_unix_seconds or is_unix_millis:- return True-- return False-- def analyze_crushability(- self,- items: list[dict],- field_stats: dict[str, FieldStats],- ) -> CrushabilityAnalysis:- """Analyze whether it's SAFE to crush this array.-- The key insight: High variability + No importance signal = DON'T CRUSH.-- We use STATISTICAL detection (no hardcoded field names):- 1. ID fields detected by uniqueness + sequential/UUID/entropy patterns- 2. Score fields detected by bounded range + sorted order- 3. Error items detected by structural outliers (rare fields, rare status values)- 4. Numeric anomalies (importance signal)- 5. Low uniqueness (safe to sample)-- Returns:- CrushabilityAnalysis with decision and reasoning.- """- signals_present: list[str] = []- signals_absent: list[str] = []-- # 1. Detect ID field STATISTICALLY (no hardcoded field names)- id_field_name = None- id_uniqueness = 0.0- id_confidence = 0.0- for name, stats in field_stats.items():- values = [item.get(name) for item in items if isinstance(item, dict)]- is_id, confidence = _detect_id_field_statistically(stats, values)- if is_id and confidence > id_confidence:- id_field_name = name- id_uniqueness = stats.unique_ratio- id_confidence = confidence-- has_id_field = id_field_name is not None and id_confidence >= 0.7-- # 2. Detect score/rank field STATISTICALLY (no hardcoded field names)- has_score_field = False- for name, stats in field_stats.items():- is_score, confidence = _detect_score_field_statistically(stats, items)- if is_score:- has_score_field = True- signals_present.append(f"score_field:{name}(conf={confidence:.2f})")- break- if not has_score_field:- signals_absent.append("score_field")-- # 3. Detect error items via STRUCTURAL OUTLIERS (no hardcoded keywords)- outlier_indices = _detect_structural_outliers(items)- structural_outlier_count = len(outlier_indices)-- if structural_outlier_count > 0:- signals_present.append(f"structural_outliers:{structural_outlier_count}")- else:- signals_absent.append("structural_outliers")-- # 3b. Also detect errors via keywords in content (for log/message-style data)- # This catches errors that are in the content but not structural outliers- # (e.g., Slack messages where error is in the text field)- error_keyword_indices = _detect_error_items_for_preservation(items)- keyword_error_count = len(error_keyword_indices)-- if keyword_error_count > 0 and structural_outlier_count == 0:- signals_present.append(f"error_keywords:{keyword_error_count}")-- # Combined error count for crushability analysis- error_count = max(structural_outlier_count, keyword_error_count)-- # 4. Count numeric anomalies (importance signal)- anomaly_count = 0- anomaly_indices: set[int] = set()- for stats in field_stats.values():- if stats.field_type == "numeric" and stats.mean_val is not None and stats.variance:- std = stats.variance**0.5- if std > 0:- threshold = self.config.variance_threshold * std- for i, item in enumerate(items):- val = item.get(stats.name)- if isinstance(val, int | float):- if abs(val - stats.mean_val) > threshold:- anomaly_indices.add(i)-- anomaly_count = len(anomaly_indices)- if anomaly_count > 0:- signals_present.append(f"anomalies:{anomaly_count}")- else:- signals_absent.append("anomalies")-- # 5. Compute average string uniqueness (EXCLUDING statistically-detected ID fields)- string_stats = [- s for s in field_stats.values() if s.field_type == "string" and s.name != id_field_name- ]- avg_string_uniqueness = (- statistics.mean(s.unique_ratio for s in string_stats) if string_stats else 0.0- )-- # Compute uniqueness of non-ID numeric fields- non_id_numeric_stats = [- s for s in field_stats.values() if s.field_type == "numeric" and s.name != id_field_name- ]- avg_non_id_numeric_uniqueness = (- statistics.mean(s.unique_ratio for s in non_id_numeric_stats)- if non_id_numeric_stats- else 0.0- )-- # Combined uniqueness metric (including ID fields)- max_uniqueness = max(avg_string_uniqueness, id_uniqueness, 0.0)-- # Non-ID content uniqueness (for detecting repetitive content with unique IDs)- non_id_content_uniqueness = max(avg_string_uniqueness, avg_non_id_numeric_uniqueness)-- # 6. Check for change points (importance signal for time series)- has_change_points = any(- stats.change_points for stats in field_stats.values() if stats.field_type == "numeric"- )- if has_change_points:- signals_present.append("change_points")-- # DECISION LOGIC- has_any_signal = len(signals_present) > 0-- # Case 0: Repetitive content with unique IDs- # If all non-ID fields are nearly constant, data is safe to sample- # even if there's a unique ID field (e.g., status="success" for all items)- if non_id_content_uniqueness < 0.1 and has_id_field:- signals_present.append("repetitive_content")- return CrushabilityAnalysis(- crushable=True,- confidence=0.85,- reason="repetitive_content_with_ids",- signals_present=signals_present,- signals_absent=signals_absent,- has_id_field=has_id_field,- id_uniqueness=id_uniqueness,- avg_string_uniqueness=avg_string_uniqueness,- has_score_field=has_score_field,- error_item_count=error_count,- anomaly_count=anomaly_count,- )-- # Case 1: Low uniqueness - safe to sample (data is repetitive)- if max_uniqueness < 0.3:- return CrushabilityAnalysis(- crushable=True,- confidence=0.9,- reason="low_uniqueness_safe_to_sample",- signals_present=signals_present,- signals_absent=signals_absent,- has_id_field=has_id_field,- id_uniqueness=id_uniqueness,- avg_string_uniqueness=avg_string_uniqueness,- has_score_field=has_score_field,- error_item_count=error_count,- anomaly_count=anomaly_count,- )-- # Case 2: High uniqueness + ID field + NO signal = DON'T CRUSH- # This is the critical case: DB results, file listings, user lists- if has_id_field and max_uniqueness > 0.8 and not has_any_signal:- return CrushabilityAnalysis(- crushable=False,- confidence=0.85,- reason="unique_entities_no_signal",- signals_present=signals_present,- signals_absent=signals_absent,- has_id_field=has_id_field,- id_uniqueness=id_uniqueness,- avg_string_uniqueness=avg_string_uniqueness,- has_score_field=has_score_field,- error_item_count=error_count,- anomaly_count=anomaly_count,- )-- # Case 3: High uniqueness + has signal = CRUSH using signal- if max_uniqueness > 0.8 and has_any_signal:- return CrushabilityAnalysis(- crushable=True,- confidence=0.7,- reason="unique_entities_with_signal",- signals_present=signals_present,- signals_absent=signals_absent,- has_id_field=has_id_field,- id_uniqueness=id_uniqueness,- avg_string_uniqueness=avg_string_uniqueness,- has_score_field=has_score_field,- error_item_count=error_count,- anomaly_count=anomaly_count,- )-- # Case 4: Medium uniqueness + no signal = be cautious, don't crush- if not has_any_signal:- return CrushabilityAnalysis(- crushable=False,- confidence=0.6,- reason="medium_uniqueness_no_signal",- signals_present=signals_present,- signals_absent=signals_absent,- has_id_field=has_id_field,- id_uniqueness=id_uniqueness,- avg_string_uniqueness=avg_string_uniqueness,- has_score_field=has_score_field,- error_item_count=error_count,- anomaly_count=anomaly_count,- )-- # Case 5: Medium uniqueness + has signal = crush with caution- return CrushabilityAnalysis(- crushable=True,- confidence=0.5,- reason="medium_uniqueness_with_signal",- signals_present=signals_present,- signals_absent=signals_absent,- has_id_field=has_id_field,- id_uniqueness=id_uniqueness,- avg_string_uniqueness=avg_string_uniqueness,- has_score_field=has_score_field,- error_item_count=error_count,- anomaly_count=anomaly_count,- )-- def _select_strategy(- self,- field_stats: dict[str, FieldStats],- pattern: str,- item_count: int,- crushability: CrushabilityAnalysis | None = None,- ) -> CompressionStrategy:- """Select optimal compression strategy based on analysis."""- if item_count < self.config.min_items_to_analyze:- return CompressionStrategy.NONE-- # CRITICAL: Check crushability first- if crushability is not None and not crushability.crushable:- return CompressionStrategy.SKIP-- if pattern == "time_series":- # Check if there are change points worth preserving- numeric_fields = [v for v in field_stats.values() if v.field_type == "numeric"]- has_change_points = any(f.change_points for f in numeric_fields)- if has_change_points:- return CompressionStrategy.TIME_SERIES-- if pattern == "logs":- # Check if messages are clusterable (low-medium uniqueness)- message_field = next(- (v for k, v in field_stats.items() if "message" in k.lower()), None- )- if message_field and message_field.unique_ratio < 0.5:- return CompressionStrategy.CLUSTER_SAMPLE-- if pattern == "search_results":- return CompressionStrategy.TOP_N-- # Default: smart sampling- return CompressionStrategy.SMART_SAMPLE-- def _estimate_reduction(- self, field_stats: dict[str, FieldStats], strategy: CompressionStrategy, item_count: int- ) -> float:- """Estimate token reduction ratio."""- if strategy == CompressionStrategy.NONE:- return 0.0-- # Count constant fields (will be factored out)- constant_ratio = sum(1 for v in field_stats.values() if v.is_constant) / len(field_stats)-- # Estimate based on strategy- base_reduction = {- CompressionStrategy.TIME_SERIES: 0.7,- CompressionStrategy.CLUSTER_SAMPLE: 0.8,- CompressionStrategy.TOP_N: 0.6,- CompressionStrategy.SMART_SAMPLE: 0.5,- }.get(strategy, 0.3)-- # Adjust for constants- reduction = base_reduction + (constant_ratio * 0.2)-- return min(reduction, 0.95)---class SmartCrusher(Transform):- """- Intelligent tool output compression using statistical analysis.-- Unlike fixed-rule crushing, SmartCrusher:- 1. Analyzes JSON structure and computes field statistics- 2. Detects data patterns (time series, logs, search results)- 3. Identifies constant fields to factor out- 4. Finds change points in numeric data to preserve- 5. Applies optimal compression strategy per data type- 6. Uses RelevanceScorer for semantic matching of user queries-- This results in higher compression with lower information loss.- """-- name = "smart_crusher"-- def __init__(- self,- config: SmartCrusherConfig | None = None,- relevance_config: RelevanceScorerConfig | None = None,- scorer: RelevanceScorer | None = None,- ccr_config: CCRConfig | None = None,- ):- self.config = config or SmartCrusherConfig()- self.analyzer = SmartAnalyzer(self.config)-- # CCR (Compress-Cache-Retrieve) configuration- # When no ccr_config provided, default to caching enabled but markers disabled- # This maintains backward compatibility - callers must opt-in to markers- if ccr_config is None:- self._ccr_config = CCRConfig(- enabled=True, # Still cache for potential retrieval- inject_retrieval_marker=False, # Don't break JSON parsing by default- )- else:- self._ccr_config = ccr_config- self._compression_store: CompressionStore | None = None-- # Feedback loop for learning compression patterns- self._feedback: CompressionFeedback | None = None-- # CRITICAL FIX: Lock for thread-safe lazy initialization- # Without this, multiple threads could call _get_* methods simultaneously- # and potentially create redundant initialization calls.- self._lazy_init_lock = threading.Lock()-- # Initialize relevance scorer- if scorer is not None:- self._scorer = scorer- else:- rel_config = relevance_config or RelevanceScorerConfig()- # Build kwargs based on tier - BM25 params only apply to bm25 tier- scorer_kwargs = {}- if rel_config.tier == "bm25":- scorer_kwargs = {"k1": rel_config.bm25_k1, "b": rel_config.bm25_b}- elif rel_config.tier == "hybrid":- scorer_kwargs = {- "alpha": rel_config.hybrid_alpha,- "adaptive": rel_config.adaptive_alpha,- }- self._scorer = create_scorer(tier=rel_config.tier, **scorer_kwargs)- # Use threshold from config, or default from RelevanceScorerConfig- rel_cfg = relevance_config or RelevanceScorerConfig()- self._relevance_threshold = rel_cfg.relevance_threshold-- # Initialize AnchorSelector for dynamic position-based preservation- anchor_config = self.config.anchor if hasattr(self.config, "anchor") else AnchorConfig()- self._anchor_selector = AnchorSelector(anchor_config)-- # NOTE: Error detection now uses structural outlier detection (_detect_structural_outliers)- # instead of hardcoded keywords. This scales to any data domain.-- def _map_to_anchor_pattern(self, strategy: CompressionStrategy) -> AnchorDataPattern:- """Map SmartCrusher compression strategy to AnchorSelector data pattern.-- Args:- strategy: The detected compression strategy.-- Returns:- Corresponding AnchorDataPattern for anchor selection.- """- return {- CompressionStrategy.TIME_SERIES: AnchorDataPattern.TIME_SERIES,- CompressionStrategy.TOP_N: AnchorDataPattern.SEARCH_RESULTS,- CompressionStrategy.CLUSTER_SAMPLE: AnchorDataPattern.LOGS,- CompressionStrategy.SMART_SAMPLE: AnchorDataPattern.GENERIC,- }.get(strategy, AnchorDataPattern.GENERIC)-- def crush(self, content: str, query: str = "", bias: float = 1.0) -> CrushResult:- """Crush content string directly (for use by ContentRouter).-- This is a simplified interface for compressing a single content string,- used by ContentRouter when routing JSON arrays to SmartCrusher.-- Args:- content: JSON string content to compress.- query: Query context for relevance-based compression.- bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer).-- Returns:- CrushResult with compressed content and metadata.- """- compressed, was_modified, analysis_info = self._smart_crush_content(- content, query_context=query, bias=bias- )- return CrushResult(- compressed=compressed,- original=content,- was_modified=was_modified,- strategy=analysis_info or "passthrough",- )-- def _get_compression_store(self) -> CompressionStore:- """Get the compression store for CCR (lazy initialization).-- CRITICAL FIX: Thread-safe double-checked locking pattern.- """- if self._compression_store is None:- with self._lazy_init_lock:- # Double-check after acquiring lock- if self._compression_store is None:- self._compression_store = get_compression_store(- max_entries=self._ccr_config.store_max_entries,- default_ttl=self._ccr_config.store_ttl_seconds,- )- return self._compression_store-- def _get_feedback(self) -> CompressionFeedback:- """Get the feedback analyzer (lazy initialization).-- CRITICAL FIX: Thread-safe double-checked locking pattern.- """- if self._feedback is None:- with self._lazy_init_lock:- if self._feedback is None:- self._feedback = get_compression_feedback()- return self._feedback-- def _get_telemetry(self) -> TelemetryCollector:- """Get the telemetry collector (lazy initialization).-- CRITICAL FIX: Thread-safe double-checked locking pattern.- """- # Use getattr to avoid hasattr race condition- if getattr(self, "_telemetry", None) is None:- with self._lazy_init_lock:- if getattr(self, "_telemetry", None) is None:- self._telemetry = get_telemetry_collector()- return self._telemetry-- def _get_toin(self) -> ToolIntelligenceNetwork:- """Get the TOIN instance (lazy initialization).-- CRITICAL FIX: Thread-safe double-checked locking pattern.- """- # Use getattr to avoid hasattr race condition- if getattr(self, "_toin", None) is None:- with self._lazy_init_lock:- if getattr(self, "_toin", None) is None:- self._toin = get_toin()- return self._toin-- def _record_telemetry(- self,- items: list[dict],- result: list,- analysis: ArrayAnalysis,- plan: CompressionPlan,- tool_name: str | None = None,- ) -> None:- """Record compression telemetry for the data flywheel.-- This collects anonymized statistics about compression patterns to- enable cross-user learning and improve compression over time.-- Privacy guarantees:- - No actual data values are stored- - Tool names can be hashed- - Only structural patterns are captured- """- try:- telemetry = self._get_telemetry()-- # Calculate what was kept- kept_first_n = sum(1 for i in plan.keep_indices if i < 3)- kept_last_n = sum(1 for i in plan.keep_indices if i >= len(items) - 2)-- # Count error items in result- error_indices = set(_detect_error_items_for_preservation(items))- kept_errors = sum(1 for i in plan.keep_indices if i in error_indices)-- # Count anomalies (approximate from change points)- anomaly_count = 0- for stats in analysis.field_stats.values():- if stats.change_points:- anomaly_count += len(stats.change_points)- kept_anomalies = min(anomaly_count, len(plan.keep_indices))-- # Crushability info- crushability_score = None- crushability_reason = None- if analysis.crushability:- crushability_score = analysis.crushability.confidence- crushability_reason = analysis.crushability.reason-- # Record the event- telemetry.record_compression(- items=items[:100], # Sample for structure analysis- original_count=len(items),- compressed_count=len(result),- original_tokens=0, # Not available here- compressed_tokens=0, # Not available here- strategy=analysis.recommended_strategy.value,- tool_name=tool_name,- strategy_reason=analysis.detected_pattern,- crushability_score=crushability_score,- crushability_reason=crushability_reason,- kept_first_n=kept_first_n,- kept_last_n=kept_last_n,- kept_errors=kept_errors,- kept_anomalies=kept_anomalies,- kept_by_relevance=0, # Would need to track separately- kept_by_score=0, # Would need to track separately- )- except Exception:- # Telemetry should never break compression- pass-- def _deduplicate_indices_by_content(- self,- keep_indices: set[int],- items: list[dict],- ) -> set[int]:- """Deduplicate indices by content hash, preferring lower indices.-- When multiple indices contain identical content, only the lowest index- is kept. This maximizes information density in the compressed output.-- Enterprise considerations:- - Thread-safe: No shared state modified- - Performance: O(n) single pass with hash map- - Memory: O(n) for hash storage (16-char hashes)- - Fault-tolerant: Serialization errors preserve the item-- Args:- keep_indices: Set of indices to deduplicate.- items: The items array for content lookup.-- Returns:- Deduplicated set of indices (may be smaller than input).- """- if not keep_indices:- return keep_indices-- # Track first occurrence of each content hash- # Using dict[hash -> lowest_index] for O(1) lookup- seen_hashes: dict[str, int] = {}- duplicates_removed = 0-- # Process in sorted order to ensure deterministic "lowest index wins"- for idx in sorted(keep_indices):- # Bounds check- if idx < 0 or idx >= len(items):- continue-- item = items[idx]-- # Compute content hash- try:- if isinstance(item, dict):- # Canonical JSON serialization for consistent hashing- content = json.dumps(item, sort_keys=True, default=str)- else:- # Non-dict items: use string representation- content = str(item)- item_hash = hashlib.md5(content.encode()).hexdigest()[:16] # nosec B324- except (TypeError, ValueError, RecursionError) as e:- # Serialization failed - keep the item (fail-safe)- logger.debug("Dedup hash failed for item at index %d: %s. Keeping item.", idx, e)- # Use index as unique "hash" to ensure item is kept- item_hash = f"__idx_{idx}__"-- # First occurrence wins- if item_hash not in seen_hashes:- seen_hashes[item_hash] = idx- else:- duplicates_removed += 1-- # Log deduplication stats for observability- if duplicates_removed > 0:- logger.debug(- "Content deduplication removed %d duplicate items from %d candidates "- "(%.1f%% reduction). Unique items: %d",- duplicates_removed,- len(keep_indices),- 100 * duplicates_removed / len(keep_indices),- len(seen_hashes),- )-- return set(seen_hashes.values())-- def _fill_remaining_slots(- self,- keep_indices: set[int],- items: list[dict],- n: int,- effective_max: int,- ) -> set[int]:- """Fill remaining slots with unique items when under budget.-- When deduplication reduces keep_indices below effective_max, this method- fills the remaining slots with diverse items from the array. It:- 1. Computes content hashes for already-kept items- 2. Scans array for items with unique content not yet kept- 3. Distributes new items evenly across the array for coverage-- Enterprise considerations:- - Thread-safe: No shared state modified- - Performance: O(n) for hash computation, O(n) for filling- - Memory: O(k) where k = len(keep_indices) for hash storage- - Deterministic: Same input always produces same output-- Args:- keep_indices: Current set of indices to keep (already deduplicated).- items: The full items array.- n: Total number of items.- effective_max: Target maximum items.-- Returns:- Updated set of indices, filled up to effective_max with unique items.- """- remaining_slots = effective_max - len(keep_indices)- if remaining_slots <= 0:- return keep_indices-- # Build set of content hashes for items we're already keeping- seen_hashes: set[str] = set()- for idx in keep_indices:- if 0 <= idx < n:- item = items[idx]- try:- if isinstance(item, dict):- content = json.dumps(item, sort_keys=True, default=str)- else:- content = str(item)- seen_hashes.add(hashlib.md5(content.encode()).hexdigest()[:16]) # nosec B324- except (TypeError, ValueError, RecursionError):- pass # Skip hash computation failures-- # Find candidate indices not in keep_indices- candidates = [i for i in range(n) if i not in keep_indices]- if not candidates:- return keep_indices-- # Distribute selection evenly across the array for coverage- # Use stride-based sampling to avoid clustering- result = keep_indices.copy()- added = 0-- # Calculate step size for even distribution- step = max(1, len(candidates) // (remaining_slots + 1))-- # First pass: evenly distributed unique items- for start_offset in range(step):- if added >= remaining_slots:- break- for i in range(start_offset, len(candidates), step):- if added >= remaining_slots:- break- idx = candidates[i]- item = items[idx]-- # Check if this item's content is unique- try:- if isinstance(item, dict):- content = json.dumps(item, sort_keys=True, default=str)- else:- content = str(item)- item_hash = hashlib.md5(content.encode()).hexdigest()[:16] # nosec B324- except (TypeError, ValueError, RecursionError):- # Hash failure - use index as unique hash (fail-safe)- item_hash = f"__idx_{idx}__"-- if item_hash not in seen_hashes:- result.add(idx)- seen_hashes.add(item_hash)- added += 1-- if added > 0:- logger.debug(- "Filled %d remaining slots (from %d to %d items) after deduplication",- added,- len(keep_indices),- len(result),- )-- return result-- def _prioritize_indices(- self,- keep_indices: set[int],- items: list[dict],- n: int,- analysis: ArrayAnalysis | None = None,- max_items: int | None = None,- field_semantics: dict[str, FieldSemantics] | None = None,- ) -> set[int]:- """Prioritize indices when we exceed max_items, ALWAYS keeping critical items.-- Priority order:- 1. ALL error items (non-negotiable) - items with error keywords- 2. ALL structural outliers (non-negotiable) - items with rare fields/status values- 3. ALL numeric anomalies (non-negotiable) - e.g., unusual values like 999999- 4. ALL items with important values (learned) - TOIN field semantics- 5. First 3 items (context)- 6. Last 2 items (context)- 7. Other important items by index order-- Uses BOTH keyword detection (for preservation guarantee) AND statistical detection,- PLUS learned field semantics from TOIN for zero-latency signal detection.-- HIGH FIX: Note that this function may return MORE items than effective_max- when critical items (errors, outliers, anomalies) exceed the limit. This is- intentional to preserve the quality guarantee. A warning is logged when this- happens to help diagnose cases where compression is less effective than expected.-- Args:- keep_indices: Initial set of indices to keep.- items: The items being compressed.- n: Total number of items.- analysis: Optional analysis results for anomaly detection.- max_items: Thread-safe max items limit (defaults to config value).- field_semantics: Optional learned field semantics from TOIN.-- Returns:- Set of indices to keep (may exceed max_items if critical items require it).- """- # Use provided max_items or fall back to config- effective_max = max_items if max_items is not None else self.config.max_items_after_crush-- # === ENTERPRISE FIX: Content-based deduplication ===- # Multiple preservation mechanisms (anchors, anomalies, outliers, etc.) can add- # the same item multiple times by index. More critically, different INDICES can- # contain IDENTICAL content (e.g., 10 identical status messages at indices 0-9).- # Without deduplication, we waste slots on redundant information.- #- # This deduplication runs FIRST, before any other logic, to ensure:- # 1. Budget calculations work with unique items only- # 2. Critical item detection doesn't double-count- # 3. Final output maximizes information density- #- # Performance: O(n) where n = len(keep_indices), single pass with hash map- # Memory: O(n) for hash storage, hashes are 16 chars each- if self.config.dedup_identical_items:- keep_indices = self._deduplicate_indices_by_content(keep_indices, items)-- # === ENTERPRISE FIX: Fill up to max_items when under budget ===- # After deduplication, we may have fewer unique items than max_items.- # Instead of returning a sparse result, fill remaining slots with- # diverse items from the array. This maximizes information density.- if len(keep_indices) < effective_max and len(keep_indices) < n:- keep_indices = self._fill_remaining_slots(keep_indices, items, n, effective_max)-- if len(keep_indices) <= effective_max:- return keep_indices-- # Use provided field_semantics or fall back to thread-local (set by _crush_array)- effective_field_semantics = field_semantics or getattr(- getattr(self, "_thread_local", None), "field_semantics", None- )-- # Identify error items using KEYWORD detection (preservation guarantee)- # This ensures ALL error items are kept, regardless of frequency- error_indices = set(_detect_error_items_for_preservation(items))-- # Identify structural outlier indices using STATISTICAL detection- # (items with rare fields or rare status values)- outlier_indices = set(_detect_structural_outliers(items))-- # Identify numeric anomalies (MUST keep ALL of them)- anomaly_indices = set()- if analysis and analysis.field_stats:- for field_name, stats in analysis.field_stats.items():- if stats.field_type == "numeric" and stats.mean_val is not None and stats.variance:- std = stats.variance**0.5- if std > 0:- threshold = self.config.variance_threshold * std- for i, item in enumerate(items):- val = item.get(field_name)- if isinstance(val, int | float):- if abs(val - stats.mean_val) > threshold:- anomaly_indices.add(i)-- # === TOIN Evolution: Identify items with important values (learned) ===- # Uses learned field semantics for zero-latency signal detection- learned_important_indices: set[int] = set()- if effective_field_semantics:- learned_important_indices = set(- _detect_items_by_learned_semantics(items, effective_field_semantics)- )-- # Start with all critical items (these are non-negotiable)- # Error items are ALWAYS preserved (quality guarantee)- prioritized = error_indices | outlier_indices | anomaly_indices | learned_important_indices-- # HIGH FIX: Log warning if critical items alone exceed the limit- # This helps diagnose why compression may be less effective than expected- critical_count = len(prioritized)- if critical_count > effective_max:- logger.warning(- "Critical items (%d) exceed max_items (%d): errors=%d outliers=%d anomalies=%d learned=%d. "- "Quality guarantee takes precedence - keeping all critical items.",- critical_count,- effective_max,- len(error_indices),- len(outlier_indices),- len(anomaly_indices),- len(learned_important_indices),- )-- # Add first/last items if we have room- remaining_slots = effective_max - len(prioritized)- if remaining_slots > 0:- # First 3 items- for i in range(min(3, n)):- if i not in prioritized and remaining_slots > 0:- prioritized.add(i)- remaining_slots -= 1- # Last 2 items- for i in range(max(0, n - 2), n):- if i not in prioritized and remaining_slots > 0:- prioritized.add(i)- remaining_slots -= 1-- # Fill remaining slots with other important indices (by index order)- if remaining_slots > 0:- other_indices = sorted(keep_indices - prioritized)- for i in other_indices:- if remaining_slots <= 0:- break- prioritized.add(i)- remaining_slots -= 1-- return prioritized-- def should_apply(- self,- messages: list[dict[str, Any]],- tokenizer: Tokenizer,- **kwargs: Any,- ) -> bool:- """Check if any tool messages would benefit from smart crushing."""- if not self.config.enabled:- return False-- for msg in messages:- # OpenAI style: role="tool"- if msg.get("role") == "tool":- content = msg.get("content", "")- if isinstance(content, str):- tokens = tokenizer.count_text(content)- if tokens > self.config.min_tokens_to_crush:- # Check if it's JSON with arrays- parsed, success = safe_json_loads(content)- if success and self._has_crushable_arrays(parsed):- return True-- # Anthropic style: role="user" with tool_result content blocks- content = msg.get("content")- if isinstance(content, list):- for block in content:- if isinstance(block, dict) and block.get("type") == "tool_result":- tool_content = block.get("content", "")- if isinstance(tool_content, str):- tokens = tokenizer.count_text(tool_content)- if tokens > self.config.min_tokens_to_crush:- parsed, success = safe_json_loads(tool_content)- if success and self._has_crushable_arrays(parsed):- return True-- return False-- def _has_crushable_arrays(self, data: Any, depth: int = 0) -> bool:- """Check if data contains arrays large enough to crush.-- Accepts arrays of ANY homogeneous type (dicts, strings, numbers, etc.)- as well as mixed-type arrays. Bool-only arrays are excluded (not useful- to compress).- """- if depth > 5:- return False-- if isinstance(data, list):- if len(data) >= self.config.min_items_to_analyze:- arr_type = _classify_array(data)- if arr_type not in (ArrayType.EMPTY, ArrayType.BOOL_ARRAY):- return True- for item in data[:10]: # Check first few items- if self._has_crushable_arrays(item, depth + 1):- return True-- elif isinstance(data, dict):- # Large objects with many keys are themselves crushable- if len(data) >= self.config.min_items_to_analyze:- return True- for value in data.values():- if self._has_crushable_arrays(value, depth + 1):- return True-- return False-- def apply(- self,- messages: list[dict[str, Any]],- tokenizer: Tokenizer,- **kwargs: Any,- ) -> TransformResult:- """Apply smart crushing to messages."""- tokens_before = tokenizer.count_messages(messages)- result_messages = deep_copy_messages(messages)- transforms_applied: list[str] = []- markers_inserted: list[str] = []- warnings: list[str] = []-- # Extract query context from recent user messages for relevance scoring- query_context = self._extract_context_from_messages(result_messages)-- crushed_count = 0- frozen_message_count = kwargs.get("frozen_message_count", 0)-- for msg_idx, msg in enumerate(result_messages):- # Skip frozen messages (in provider's prefix cache)- if msg_idx < frozen_message_count:- continue-- # OpenAI style- if msg.get("role") == "tool":- content = msg.get("content", "")- if not isinstance(content, str):- continue-- tokens = tokenizer.count_text(content)- if tokens <= self.config.min_tokens_to_crush:- continue-- crushed, was_modified, analysis_info = self._smart_crush_content(- content, query_context- )-- if was_modified:- original_hash = compute_short_hash(content)- marker = create_tool_digest_marker(original_hash)- msg["content"] = crushed + "\n" + marker- crushed_count += 1- markers_inserted.append(marker)- if analysis_info:- transforms_applied.append(f"smart:{analysis_info}")-- # Anthropic style- content = msg.get("content")- if isinstance(content, list):- for i, block in enumerate(content):- if not isinstance(block, dict):- continue- if block.get("type") != "tool_result":- continue-- tool_content = block.get("content", "")- if not isinstance(tool_content, str):- continue-- tokens = tokenizer.count_text(tool_content)- if tokens <= self.config.min_tokens_to_crush:- continue-- crushed, was_modified, analysis_info = self._smart_crush_content(- tool_content, query_context- )-- if was_modified:- original_hash = compute_short_hash(tool_content)- marker = create_tool_digest_marker(original_hash)- content[i]["content"] = crushed + "\n" + marker- crushed_count += 1- markers_inserted.append(marker)- if analysis_info:- transforms_applied.append(f"smart:{analysis_info}")-- if crushed_count > 0:- transforms_applied.insert(0, f"smart_crush:{crushed_count}")-- tokens_after = tokenizer.count_messages(result_messages)-- return TransformResult(- messages=result_messages,- tokens_before=tokens_before,- tokens_after=tokens_after,- transforms_applied=transforms_applied,- markers_inserted=markers_inserted,- warnings=warnings,- )-- def _extract_context_from_messages(self, messages: list[dict[str, Any]]) -> str:- """Extract query context from recent messages for relevance scoring.-- Builds a context string from:- - Recent user messages (what the user is asking about)- - Recent tool call arguments (what data was requested)-- This context is used by RelevanceScorer to determine which items- to preserve during crushing.-- Args:- messages: Full message list.-- Returns:- Context string for relevance scoring.- """- context_parts: list[str] = []-- # Look at last 5 user messages (most relevant to recent tool calls)- user_message_count = 0- for msg in reversed(messages):- if msg.get("role") == "user":- content = msg.get("content")- if isinstance(content, str):- context_parts.append(content)- elif isinstance(content, list):- # Anthropic style - extract from text blocks- for block in content:- if isinstance(block, dict) and block.get("type") == "text":- text = block.get("text", "")- if text:- context_parts.append(text)-- user_message_count += 1- if user_message_count >= 5:- break-- # Also check assistant tool_calls for function arguments- if msg.get("role") == "assistant" and msg.get("tool_calls"):- for tc in msg.get("tool_calls", []):- if isinstance(tc, dict):- func = tc.get("function", {})- args = func.get("arguments", "")- if isinstance(args, str) and args:- context_parts.append(args)-- return " ".join(context_parts)-- def _smart_crush_content(- self,- content: str,- query_context: str = "",- tool_name: str | None = None,- bias: float = 1.0,- ) -> tuple[str, bool, str]:- """- Apply smart crushing to content.-- Handles both JSON (existing SmartCrusher logic) and plain text content- (search results, logs, generic text) using specialized compressors.-- Args:- content: Content to crush (JSON or plain text).- query_context: Context string from user messages for relevance scoring.- tool_name: Name of the tool that produced this output.- bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer).-- Returns:- Tuple of (crushed_content, was_modified, analysis_info).- """- parsed, success = safe_json_loads(content)- if not success:- # Not JSON - pass through unchanged- # Text compression utilities (SearchCompressor, LogCompressor, TextCompressor)- # are available as standalone tools for applications to use explicitly- return content, False, ""-- # Recursively process and crush arrays- crushed, info, ccr_markers = self._process_value(- parsed, query_context=query_context, tool_name=tool_name, bias=bias- )-- result = safe_json_dumps(crushed, indent=None)- was_modified = result != content.strip()-- # CCR: Inject retrieval markers if compression happened and CCR is enabled- if was_modified and ccr_markers and self._ccr_config.inject_retrieval_marker:- for marker_data in ccr_markers:- if len(marker_data) == 4:- ccr_hash, original_count, compressed_count, dropped_summary = marker_data- else:- ccr_hash, original_count, compressed_count = marker_data- dropped_summary = ""- summary_str = f" Omitted: {dropped_summary}." if dropped_summary else ""- # Escape { } in summary to prevent .format() errors- safe_summary = summary_str.replace("{", "{{").replace("}", "}}")- ttl_seconds = getattr(self._ccr_config, "store_ttl_seconds", 300)- marker = self._ccr_config.marker_template.format(- original_count=original_count,- compressed_count=compressed_count,- hash=ccr_hash,- summary=safe_summary,- ttl_minutes=max(1, ttl_seconds // 60),- )- result += marker-- return result, was_modified, info-- # Maximum recursion depth for nested JSON processing.- # Prevents RecursionError on adversarial/deeply-nested input.- _MAX_PROCESS_DEPTH = 50-- def _process_value(- self,- value: Any,- depth: int = 0,- query_context: str = "",- tool_name: str | None = None,- bias: float = 1.0,- ) -> tuple[Any, str, list[tuple[str, int, int]]]:- """Recursively process a value, crushing arrays where appropriate.-- Returns:- Tuple of (processed_value, info_string, ccr_markers).- ccr_markers is a list of (hash, original_count, compressed_count, summary) tuples.- """- # Guard against deeply nested JSON causing RecursionError- if depth >= self._MAX_PROCESS_DEPTH:- return value, "", []-- info_parts = []- ccr_markers: list[tuple] = []-- if isinstance(value, list):- if len(value) >= self.config.min_items_to_analyze:- arr_type = _classify_array(value)-- if arr_type == ArrayType.DICT_ARRAY:- # Existing path — dict arrays (battle-tested, unchanged)- crushed, strategy, ccr_hash, dropped_summary = self._crush_array(- value, query_context, tool_name, bias=bias- )- info_parts.append(f"{strategy}({len(value)}->{len(crushed)})")- if ccr_hash:- ccr_markers.append((ccr_hash, len(value), len(crushed), dropped_summary))- return crushed, ",".join(info_parts), ccr_markers-- elif arr_type == ArrayType.STRING_ARRAY:- crushed, strategy = self._crush_string_array(value, bias=bias)- info_parts.append(f"{strategy}({len(value)}->{len(crushed)})")- return crushed, ",".join(info_parts), ccr_markers-- elif arr_type == ArrayType.NUMBER_ARRAY:- crushed, strategy = self._crush_number_array(value, bias=bias)- if isinstance(crushed, list):- info_parts.append(f"{strategy}({len(value)}->{len(crushed)})")- else:- info_parts.append(f"{strategy}({len(value)}->summary)")- return crushed, ",".join(info_parts), ccr_markers-- elif arr_type == ArrayType.MIXED_ARRAY:- crushed, strategy = self._crush_mixed_array(- value, query_context, tool_name, bias=bias- )- info_parts.append(f"{strategy}({len(value)}->{len(crushed)})")- return crushed, ",".join(info_parts), ccr_markers-- # NESTED_ARRAY, BOOL_ARRAY, EMPTY — fall through to recursive-- # Not crushable or below threshold — process items recursively- processed = []- for item in value:- p_item, p_info, p_markers = self._process_value(- item, depth + 1, query_context, tool_name, bias=bias- )- processed.append(p_item)- if p_info:- info_parts.append(p_info)- ccr_markers.extend(p_markers)- return processed, ",".join(info_parts), ccr_markers-- elif isinstance(value, dict):- # First: recurse into values to compress nested arrays- processed_dict: dict[str, Any] = {}- for k, v in value.items():- p_val, p_info, p_markers = self._process_value(- v, depth + 1, query_context, tool_name, bias=bias- )- processed_dict[k] = p_val- if p_info:- info_parts.append(p_info)- ccr_markers.extend(p_markers)-- # Second: if the object itself has many keys, compress at key level- if len(processed_dict) >= self.config.min_items_to_analyze:- crushed_dict, strategy = self._crush_object(processed_dict, bias=bias)- if strategy != "object:passthrough":- info_parts.append(strategy)- return crushed_dict, ",".join(info_parts), ccr_markers-- return processed_dict, ",".join(info_parts), ccr_markers-- else:- return value, "", []-- def _crush_array(- self,- items: list[dict],- query_context: str = "",- tool_name: str | None = None,- bias: float = 1.0,- ) -> tuple[list, str, str | None, str]:- """Crush an array using statistical analysis and relevance scoring.-- IMPORTANT: If crushability analysis determines it's not safe to crush- (high variability + no importance signal), returns original array unchanged.-- TOIN-aware: Consults the Tool Output Intelligence Network for cross-user- learned patterns. High retrieval rate across all users → compress less.-- Feedback-aware: Uses learned patterns to adjust compression aggressiveness.- High retrieval rate for a tool → compress less aggressively.-- Args:- items: List of dict items to compress.- query_context: Context string from user messages for relevance scoring.- tool_name: Name of the tool that produced this output.- bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer).-- Returns:- Tuple of (crushed_items, strategy_info, ccr_hash, dropped_summary).- ccr_hash is the hash for retrieval if CCR is enabled, None otherwise.- dropped_summary is a categorical summary of what was dropped.- """- # BOUNDARY CHECK: Use adaptive sizing instead of hardcoded limit- # compute_optimal_k handles trivial cases (n <= 8 → keep all)- from .adaptive_sizer import compute_optimal_k-- item_strings = [json.dumps(item, default=str) for item in items]- adaptive_k = compute_optimal_k(- item_strings,- bias=bias,- min_k=3,- max_k=self.config.max_items_after_crush if self.config.max_items_after_crush else None,- )-- if len(items) <= adaptive_k:- # All items kept (high diversity or small array).- # Instead of passing through unchanged, try to compress the TEXT- # WITHIN each item — reduce token count without losing any item.- compressed_items = _compress_text_within_items(items, query_context)- if compressed_items is not items:- return compressed_items, "compress_within:diversity", None, ""- return items, "none:adaptive_at_limit", None, ""-- # Get feedback hints if enabled- # THREAD-SAFETY: Use a local effective_max_items instead of mutating shared config- effective_max_items = adaptive_k- hints_applied = False- toin_hint_applied = False-- # Create ToolSignature for TOIN lookup- tool_signature = ToolSignature.from_items(items)-- # TOIN: Get cross-user learned recommendations- toin = self._get_toin()- toin_hint = toin.get_recommendation(tool_signature, query_context)-- # Log TOIN hint details- logger.debug(- "TOIN hint: source=%s, confidence=%.2f, skip=%s, max_items=%d",- toin_hint.source,- toin_hint.confidence,- toin_hint.skip_compression,- toin_hint.max_items,- )-- if toin_hint.skip_compression:- return items, f"skip:toin({toin_hint.reason})", None, ""-- # Apply TOIN recommendations if from network or local learning- toin_preserve_fields: list[str] = []- toin_recommended_strategy: str | None = None- toin_compression_level: str | None = None- # LOW FIX #21: Use configurable threshold instead of hardcoded 0.5- if (- toin_hint.source in ("network", "local")- and toin_hint.confidence >= self.config.toin_confidence_threshold- ):- # TOIN recommendations take precedence over local feedback- effective_max_items = toin_hint.max_items- toin_preserve_fields = toin_hint.preserve_fields # Fields to never remove- toin_hint_applied = True- # Store strategy and compression level for later use- if toin_hint.recommended_strategy != "default":- toin_recommended_strategy = toin_hint.recommended_strategy- if toin_hint.compression_level != "moderate":- toin_compression_level = toin_hint.compression_level- # Log that TOIN hint was applied- logger.debug(- "TOIN hint applied: max_items=%d, strategy=%s, compression_level=%s",- effective_max_items,- toin_recommended_strategy or "default",- toin_compression_level or "moderate",- )- elif toin_hint.source in ("network", "local"):- # Hint available but confidence too low- logger.debug(- "TOIN hint not applied: confidence %.2f < threshold %.2f",- toin_hint.confidence,- self.config.toin_confidence_threshold,- )-- # === TOIN Evolution: Extract field semantics for signal detection ===- # Store in thread-local storage for use in _prioritize_indices.- # This enables learned signal detection without changing all method signatures- # while remaining thread-safe (no cross-thread contamination).- if not hasattr(self, "_thread_local"):- self._thread_local = threading.local()- self._thread_local.field_semantics = (- toin_hint.field_semantics if toin_hint.field_semantics else None- )-- # Local feedback hints (if TOIN didn't apply)- if not toin_hint_applied and self.config.use_feedback_hints and tool_name:- feedback = self._get_feedback()- hints = feedback.get_compression_hints(tool_name)-- # Check if hints recommend skipping compression- if hints.skip_compression:- return items, f"skip:feedback({hints.reason})", None, ""-- # Adjust max_items based on feedback- if hints.suggested_items is not None:- effective_max_items = hints.suggested_items- hints_applied = True-- # Use preserve_fields from local feedback (hash them for TOIN compatibility)- # Note: CompressionFeedback stores actual field names, but _plan methods- # expect SHA256[:8] hashes for privacy-preserving comparison- if hints.preserve_fields:- toin_preserve_fields = [_hash_field_name(field) for field in hints.preserve_fields]-- # Use recommended_strategy from local feedback if not already set by TOIN- if hints.recommended_strategy and not toin_recommended_strategy:- toin_recommended_strategy = hints.recommended_strategy-- try:- # Analyze the array (includes crushability check)- analysis = self.analyzer.analyze_array(items)-- # CRITICAL: If not crushable, return original array unchanged- if analysis.recommended_strategy == CompressionStrategy.SKIP:- reason = ""- if analysis.crushability:- reason = f"skip:{analysis.crushability.reason}"- return items, reason, None, ""-- # Apply TOIN strategy recommendation if available- # TOIN learns which strategies work best from cross-user patterns- if toin_recommended_strategy:- try:- toin_strategy = CompressionStrategy(toin_recommended_strategy)- # Only override if TOIN suggests a valid non-SKIP strategy- if toin_strategy != CompressionStrategy.SKIP:- analysis.recommended_strategy = toin_strategy- except ValueError:- pass # Invalid strategy name, keep analyzer's choice-- # Apply TOIN compression level to adjust effective_max_items- if toin_compression_level:- if toin_compression_level == "none":- # Don't compress - return original- return items, "skip:toin_level_none", None, ""- elif toin_compression_level == "conservative":- # Be conservative - keep more items- effective_max_items = max(effective_max_items, min(50, len(items) // 2))- elif toin_compression_level == "aggressive":- # Be aggressive - keep fewer items- effective_max_items = min(effective_max_items, 15)-- # Create compression plan with relevance scoring- # Pass TOIN preserve_fields so items with those fields get priority- # Pass effective_max_items for thread-safe compression- # Pass item_strings to avoid redundant json.dumps across plan methods- plan = self._create_plan(- analysis,- items,- query_context,- preserve_fields=toin_preserve_fields or None,- effective_max_items=effective_max_items,- item_strings=item_strings,- )-- # Execute compression- result = self._execute_plan(plan, items, analysis)-- # CCR: Store original content for retrieval if enabled- ccr_hash = None- if (- self._ccr_config.enabled- and len(items) >= self._ccr_config.min_items_to_cache- and len(result) < len(items) # Only cache if compression actually happened- ):- store = self._get_compression_store()- # Reuse cached item_strings to avoid re-serializing- original_json = "[" + ", ".join(item_strings) + "]"- compressed_json = json.dumps(result, default=str)-- ccr_hash = store.store(- original=original_json,- compressed=compressed_json,- original_item_count=len(items),- compressed_item_count=len(result),- tool_name=tool_name,- query_context=query_context,- # CRITICAL: Pass the tool_signature_hash so retrieval events- # can be correlated with compression events in TOIN- tool_signature_hash=tool_signature.structure_hash,- compression_strategy=analysis.recommended_strategy.value,- )-- # Record compression event for feedback loop- if self.config.use_feedback_hints and tool_name:- feedback = self._get_feedback()- feedback.record_compression(- tool_name=tool_name,- original_count=len(items),- compressed_count=len(result),- strategy=analysis.recommended_strategy.value,- tool_signature_hash=tool_signature.structure_hash,- )-- # Record telemetry for data flywheel- self._record_telemetry(- items=items,- result=result,- analysis=analysis,- plan=plan,- tool_name=tool_name,- )-- # TOIN: Record compression event for cross-user learning- try:- # Calculate token counts (approximate) - reuse cached item_strings- original_tokens = sum(len(s) for s in item_strings) // 4- compressed_tokens = len(json.dumps(result, default=str)) // 4-- toin.record_compression(- tool_signature=tool_signature,- original_count=len(items),- compressed_count=len(result),- original_tokens=original_tokens,- compressed_tokens=compressed_tokens,- strategy=analysis.recommended_strategy.value,- query_context=query_context,- items=items, # Pass items for field-level semantic learning- )- except Exception:- # TOIN should never break compression- pass-- strategy_info = analysis.recommended_strategy.value- if toin_hint_applied:- toin_parts = [f"items={toin_hint.max_items}", f"conf={toin_hint.confidence:.2f}"]- if toin_recommended_strategy:- toin_parts.append(f"strategy={toin_recommended_strategy}")- if toin_compression_level and toin_compression_level != "moderate":- toin_parts.append(f"level={toin_compression_level}")- strategy_info += f"(toin:{','.join(toin_parts)})"- elif hints_applied:- strategy_info += f"(feedback:{effective_max_items})"-- # Generate categorical summary of dropped items (use indices, not identity)- from .compression_summary import summarize_dropped_items-- dropped_summary = summarize_dropped_items(- items,- result,- kept_indices=set(plan.keep_indices),- )-- # Clean up temporary instance variable- if hasattr(self, "_thread_local"):- self._thread_local.field_semantics = None- return result, strategy_info, ccr_hash, dropped_summary-- except Exception:- # Clean up temporary instance variable- if hasattr(self, "_thread_local"):- self._thread_local.field_semantics = None- # Re-raise any exceptions (removed finally block since we no longer mutate config)- raise-- # =================================================================- # Universal JSON type handlers (string, number, mixed arrays)- # =================================================================-- def _compute_k_split(- self,- items: list,- bias: float = 1.0,- item_strings: list[str] | None = None,- ) -> tuple[int, int, int, int]:- """Compute adaptive K split into first/last/importance slots.-- Uses the existing Kneedle-based adaptive_sizer for K_total, then- splits according to configurable first_fraction / last_fraction.-- Args:- items: List of items (used as fallback for serialization).- bias: Compression bias multiplier.- item_strings: Pre-computed JSON serializations to avoid redundant json.dumps.-- Returns:- (k_total, k_first, k_last, k_importance)- """- from .adaptive_sizer import compute_optimal_k-- if item_strings is None:- item_strings = [json.dumps(item, default=str) for item in items]- k_total = compute_optimal_k(- item_strings,- bias=bias,- min_k=3,- max_k=self.config.max_items_after_crush or None,- )- # BUG #4 FIX: clamp k_first and k_last so their sum never exceeds- # k_total. Without the clamp, k_total=1 produces k_first=k_last=1- # (both `max(1, round(0))`), violating max_items_after_crush.- # No-op for k_total >= 2 (the common path).- k_first = max(1, round(k_total * self.config.first_fraction))- k_first = min(k_first, k_total)- k_last = max(1, round(k_total * self.config.last_fraction))- k_last = min(k_last, max(0, k_total - k_first))- k_importance = max(0, k_total - k_first - k_last)- return k_total, k_first, k_last, k_importance-- def _crush_string_array(- self,- items: list[str],- bias: float = 1.0,- ) -> tuple[list[str], str]:- """Crush an array of strings using dedup + adaptive sampling.-- Strategy:- 1. Compute adaptive K via Kneedle algorithm- 2. Always keep: error-containing strings, first K, last K- 3. Deduplicate exact matches- 4. Fill remaining budget with diverse samples (stride-based)-- Returns:- (crushed_items, strategy_string)- """- n = len(items)- if n <= 8:- return items, "string:passthrough"-- k_total, k_first, k_last, k_importance = self._compute_k_split(items, bias)-- # Mandatory: error-containing strings (never dropped)- error_indices: set[int] = set()- for i, s in enumerate(items):- s_lower = s.lower()- for keyword in _ERROR_KEYWORDS_FOR_PRESERVATION:- if keyword in s_lower:- error_indices.add(i)- break-- # Mandatory: strings with abnormal length (anomalies)- lengths = [len(s) for s in items]- if len(lengths) > 1:- mean_len = statistics.mean(lengths)- std_len = statistics.stdev(lengths)- anomaly_indices = {- i- for i, length in enumerate(lengths)- if std_len > 0 and abs(length - mean_len) > self.config.variance_threshold * std_len- }- else:- anomaly_indices = set[int]()-- # Boundary: first K, last K- first_indices = set(range(min(k_first, n)))- last_indices = set(range(max(0, n - k_last), n))-- # Combine mandatory + boundary- keep_indices = error_indices | anomaly_indices | first_indices | last_indices-- # Dedup: among remaining candidates, skip exact duplicates- seen_strings: set[str] = set()- dedup_count = 0- for i in sorted(keep_indices):- seen_strings.add(items[i])-- # Fill remaining budget with diverse stride-based samples- remaining_budget = max(0, k_total - len(keep_indices))- if remaining_budget > 0:- stride = max(1, (n - 1) // (remaining_budget + 1))- for i in range(0, n, stride):- if len(keep_indices) >= k_total + len(error_indices) + len(anomaly_indices):- break- if i not in keep_indices:- if items[i] not in seen_strings:- keep_indices.add(i)- seen_strings.add(items[i])- else:- dedup_count += 1-- # Build output in original order- result = [items[i] for i in sorted(keep_indices)]-- strategy = f"string:adaptive({n}->{len(result)}"- if dedup_count:- strategy += f",dedup={dedup_count}"- if error_indices:- strategy += f",errors={len(error_indices)}"- strategy += ")"-- return result, strategy-- def _crush_number_array(- self,- items: list[int | float],- bias: float = 1.0,- ) -> tuple[list, str]:- """Crush an array of numbers using statistical summary + outlier preservation.-- Strategy:- 1. Compute descriptive statistics (min, max, mean, median, stddev, percentiles)- 2. Detect outliers (> variance_threshold σ from mean)- 3. Detect change points (sudden shifts in running mean)- 4. Keep: first K, last K, all outliers, change points- 5. Return kept values with a prepended stats summary string-- Returns:- (crushed_items, strategy_string) where crushed_items is a list- starting with a summary string followed by representative values.- """- n = len(items)- if n <= 8:- return items, "number:passthrough"-- # Filter out non-finite values for statistics- finite = [x for x in items if isinstance(x, int | float) and math.isfinite(x)]- if not finite:- return items, "number:no_finite"-- k_total, k_first, k_last, k_importance = self._compute_k_split(items, bias)-- # Statistics- mean_val = statistics.mean(finite)- median_val = statistics.median(finite)- std_val = statistics.stdev(finite) if len(finite) > 1 else 0.0- sorted_finite = sorted(finite)- # BUG #1 FIX: replace integer-division indexing (off-by-one- # for len < 8) with proper linear-interpolation percentile.- # `_percentile_linear` matches numpy's "linear" method exactly- # — index = q * (n - 1), interpolate between floor and ceil.- p25 = _percentile_linear(sorted_finite, 0.25)- p75 = _percentile_linear(sorted_finite, 0.75)-- # Outliers (> variance_threshold σ from mean)- outlier_indices: set[int] = set()- if std_val > 0:- for i, val in enumerate(items):- if isinstance(val, int | float) and math.isfinite(val):- if abs(val - mean_val) > self.config.variance_threshold * std_val:- outlier_indices.add(i)-- # Change points (detect sudden shifts using running difference)- change_indices: set[int] = set()- if self.config.preserve_change_points and n > 10:- window = 5- for i in range(window, n - window):- left = [- items[j]- for j in range(i - window, i)- if isinstance(items[j], int | float) and math.isfinite(items[j])- ]- right = [- items[j]- for j in range(i, i + window)- if isinstance(items[j], int | float) and math.isfinite(items[j])- ]- if left and right:- left_mean = statistics.mean(left)- right_mean = statistics.mean(right)- if (- std_val > 0- and abs(right_mean - left_mean) > self.config.variance_threshold * std_val- ):- change_indices.add(i)-- # Boundary: first K, last K- first_indices = set(range(min(k_first, n)))- last_indices = set(range(max(0, n - k_last), n))-- # Combine all- keep_indices = outlier_indices | change_indices | first_indices | last_indices-- # Fill remaining budget with stride-based samples- remaining_budget = max(0, k_total - len(keep_indices))- if remaining_budget > 0:- stride = max(1, (n - 1) // (remaining_budget + 1))- for i in range(0, n, stride):- if len(keep_indices) >= k_total + len(outlier_indices):- break- if i not in keep_indices:- keep_indices.add(i)-- # Build output: kept values only (schema-preserving — no generated text)- kept_values = [items[i] for i in sorted(keep_indices)]-- # Encode statistics into the strategy string (not the array itself)- strategy = (- f"number:adaptive({n}->{len(kept_values)}"- f",min={min(finite)},max={max(finite)}"- f",mean={mean_val:.4g},median={median_val:.4g}"- f",stddev={std_val:.4g},p25={p25:.4g},p75={p75:.4g}"- )- if outlier_indices:- strategy += f",outliers={len(outlier_indices)}"- if change_indices:- strategy += f",change_points={len(change_indices)}"- strategy += ")"-- return kept_values, strategy-- def _crush_mixed_array(- self,- items: list,- query_context: str = "",- tool_name: str | None = None,- bias: float = 1.0,- ) -> tuple[list, str]:- """Crush a mixed-type array by grouping items by type and compressing each group.-- Strategy:- 1. Group items by type (dict, str, number, list, None, bool)- 2. For each group with >= min_items_to_analyze items: compress with appropriate handler- 3. For small groups: keep all items- 4. Reassemble in original order-- Returns:- (crushed_items, strategy_string)- """- n = len(items)- if n <= 8:- return items, "mixed:passthrough"-- # Group items by type, tracking original indices- groups: dict[str, list[tuple[int, Any]]] = {}- for i, item in enumerate(items):- if isinstance(item, dict):- key = "dict"- elif isinstance(item, str):- key = "str"- elif isinstance(item, bool):- key = "bool"- elif isinstance(item, int | float):- key = "number"- elif isinstance(item, list):- key = "list"- elif item is None:- key = "none"- else:- key = "other"- groups.setdefault(key, []).append((i, item))-- # Compress each group independently- keep_indices: set[int] = set()- strategy_parts: list[str] = []-- for type_key, group_items in groups.items():- indices = [idx for idx, _ in group_items]- values = [val for _, val in group_items]-- if len(values) < self.config.min_items_to_analyze:- # Small group — keep all- keep_indices.update(indices)- continue-- if type_key == "dict":- # Use existing dict array crusher- crushed, strategy, _, _ = self._crush_array(- values, query_context, tool_name, bias=bias- )- crushed_set = {json.dumps(c, sort_keys=True, default=str) for c in crushed}- for idx, val in group_items:- if json.dumps(val, sort_keys=True, default=str) in crushed_set:- keep_indices.add(idx)- strategy_parts.append(f"dict:{len(values)}->{len(crushed)}")-- elif type_key == "str":- crushed, strategy = self._crush_string_array(values, bias=bias)- crushed_set = set(crushed)- for idx, val in group_items:- if val in crushed_set:- keep_indices.add(idx)- strategy_parts.append(f"str:{len(values)}->{len(crushed)}")-- elif type_key == "number":- # For numbers in mixed arrays, just do adaptive sampling (no summary prefix)- k_total, k_first, k_last, _ = self._compute_k_split(values, bias)- first_idx = set(indices[:k_first])- last_idx = set(indices[-k_last:])- keep_indices.update(first_idx | last_idx)- # Outliers- finite = [v for v in values if isinstance(v, int | float) and math.isfinite(v)]- if len(finite) > 1:- mean_v = statistics.mean(finite)- std_v = statistics.stdev(finite)- if std_v > 0:- for idx, val in group_items:- if isinstance(val, int | float) and math.isfinite(val):- if abs(val - mean_v) > self.config.variance_threshold * std_v:- keep_indices.add(idx)- strategy_parts.append(f"num:{len(values)}")+"""Smart JSON array crusher — Rust-backed via PyO3.++The Python implementation has been retired (Stage 3c.1b, 2026-04-27).+All array compression now goes through `headroom._core.SmartCrusher`+(built from `crates/headroom-py`). Byte-equality of the two+implementations was verified against 17 recorded fixtures+(`tests/parity/fixtures/smart_crusher/`) before the Python source was+removed; the Rust crate has its own coverage in `crates/headroom-core/`+(388 unit tests + property tests).++This module retains the public surface — `SmartCrusherConfig`,+`CrushResult`, `SmartCrusher`, `smart_crush_tool_output` — so existing+call sites keep working unchanged. The dataclasses are still pure+Python because callers use `asdict()`, `__dict__`, and dataclass+matching on them. Only the `SmartCrusher` class delegates to Rust.++The `headroom._core` extension is a hard import: there is no Python+fallback. Build it locally with `scripts/build_rust_extension.sh`+(wraps `maturin develop`) or install a prebuilt wheel.++Stage 3c.1 deliberately keeps the optional subsystems (TOIN,+feedback, CCR marker injection, telemetry) disabled in the Rust port.+The shim accepts `relevance_config`, `scorer`, and `ccr_config`+constructor args for source compatibility but does not wire them+through — they re-attach in Stage 3c.2 when those subsystems land in+Rust. CCR marker injection in `_smart_crush_content` is a Stage 3c.2+follow-up; today the Rust port never emits CCR markers, so the+disabled-path behavior is byte-equal.+"""- else:- # list, bool, none, other — keep all- keep_indices.update(indices)+from __future__ import annotations- # Reassemble in original order- result = [items[i] for i in sorted(keep_indices)]+import logging+from dataclasses import dataclass+from typing import Any- strategy = f"mixed:adaptive({n}->{len(result)},{','.join(strategy_parts)})"- return result, strategy+from ..config import CCRConfig, TransformResult+from ..tokenizer import Tokenizer+from ..utils import compute_short_hash, create_tool_digest_marker, deep_copy_messages+from .base import Transform- def _crush_object(- self,- obj: dict[str, Any],- bias: float = 1.0,- ) -> tuple[dict[str, Any], str]:- """Crush a large JSON object by selecting the most informative keys.+logger = logging.getLogger(__name__)- Treats key-value pairs as items and applies adaptive K to select which- keys to retain. Preserves schema — each kept key-value pair is exact- from the original.- Strategy:- 1. Classify each value by size (tokens) and importance- 2. Always keep: keys with small values (cheap), keys with error content- 3. Compute adaptive K on key-value representations- 4. Fill remaining budget with diverse keys (stride-based)+# ─── Public dataclasses ───────────────────────────────────────────────────- Returns:- (compressed_object, strategy_string)- """- n = len(obj)- if n <= 8:- return obj, "object:passthrough"- # Estimate tokens per key-value pair- kv_tokens: list[tuple[str, int]] = []- total_tokens = 0- for key, val in obj.items():- val_str = json.dumps(val, default=str)- tokens = len(val_str) // 4 + len(key) // 4 + 2 # rough estimate- kv_tokens.append((key, tokens))- total_tokens += tokens+@dataclass+class CrushResult:+ """Result from `SmartCrusher.crush()`.- # If already small enough, passthrough- if total_tokens < self.config.min_tokens_to_crush:- return obj, "object:passthrough"+ Used by `ContentRouter` when routing JSON arrays to `SmartCrusher`.+ """- # Compute adaptive K on key-value string representations- keys = list(obj.keys())- kv_strings = [f"{k}: {json.dumps(obj[k], default=str)}" for k in keys]+ compressed: str+ original: str+ was_modified: bool+ strategy: str = "passthrough"- from .adaptive_sizer import compute_optimal_k- k_total = compute_optimal_k(- kv_strings,- bias=bias,- min_k=3,- max_k=self.config.max_items_after_crush or None,- )+@dataclass+class SmartCrusherConfig:+ """Configuration for SmartCrusher.- if k_total >= n:- return obj, "object:passthrough"+ SCHEMA-PRESERVING: output contains only items from the original+ array. No wrappers, no generated text, no metadata keys.- # Classify keys by importance- keep_keys: set[str] = set()+ Field names + defaults match the Rust `SmartCrusherConfig` byte-for-+ byte; the shim copies these straight into the PyO3 constructor.+ """- # Always keep: keys with error-containing values- for key, val in obj.items():- val_str = json.dumps(val, default=str).lower()- for keyword in _ERROR_KEYWORDS_FOR_PRESERVATION:- if keyword in val_str:- keep_keys.add(key)- break+ enabled: bool = True+ min_items_to_analyze: int = 5+ min_tokens_to_crush: int = 200+ variance_threshold: float = 2.0+ uniqueness_threshold: float = 0.1+ similarity_threshold: float = 0.8+ max_items_after_crush: int = 15+ preserve_change_points: bool = True+ factor_out_constants: bool = False+ include_summaries: bool = False+ use_feedback_hints: bool = True+ toin_confidence_threshold: float = 0.5+ dedup_identical_items: bool = True+ first_fraction: float = 0.3+ last_fraction: float = 0.15- # Always keep: keys with small values (cheap to keep)- small_threshold = 50 # chars- for key, tokens in kv_tokens:- if tokens <= small_threshold // 4:- keep_keys.add(key)- # Boundary: first K and last K keys- k_first = max(1, round(k_total * self.config.first_fraction))- k_last = max(1, round(k_total * self.config.last_fraction))- for key in keys[:k_first]:- keep_keys.add(key)- for key in keys[-k_last:]:- keep_keys.add(key)+# ─── Rust-backed SmartCrusher ─────────────────────────────────────────────- # Fill remaining budget with stride-based diverse sampling- remaining = max(0, k_total - len(keep_keys))- if remaining > 0:- stride = max(1, (n - 1) // (remaining + 1))- for i in range(0, n, stride):- if len(keep_keys) >= k_total + len(- [- k- for k in keep_keys- if any(- kw in json.dumps(obj[k], default=str).lower()- for kw in _ERROR_KEYWORDS_FOR_PRESERVATION- )- ]- ):- break- keep_keys.add(keys[i])- # Build output preserving original key order- result = {k: obj[k] for k in keys if k in keep_keys}+class SmartCrusher(Transform):+ """Rust-backed `SmartCrusher` (via PyO3 / `headroom._core`).++ Same `__init__` and method shapes as the retired Python class —+ drop-in replacement. The `crush()` and `_smart_crush_content()`+ methods delegate every byte to Rust; `apply()` keeps the+ Transform-protocol orchestration in Python (message walking,+ digest-marker insertion, token counting) since that's mostly glue+ around the per-message compression call.+ """- strategy = f"object:adaptive({n}->{len(result)} keys)"- return result, strategy+ name = "smart_crusher"- def _create_plan(+ def __init__(self,- analysis: ArrayAnalysis,- items: list[dict],- query_context: str = "",- preserve_fields: list[str] | None = None,- effective_max_items: int | None = None,- item_strings: list[str] | None = None,- ) -> CompressionPlan:- """Create a detailed compression plan using relevance scoring.-- Args:- analysis: The array analysis results.- items: The items to compress.- query_context: Context string from user messages for relevance scoring.- preserve_fields: TOIN-learned fields that users commonly retrieve.- Items with values in these fields get higher priority.- item_strings: Pre-computed JSON serializations to avoid redundant json.dumps.- effective_max_items: Thread-safe max items limit (defaults to config value).- """- # Use provided effective_max_items or fall back to config- max_items = (- effective_max_items- if effective_max_items is not None- else self.config.max_items_after_crush+ config: SmartCrusherConfig | None = None,+ relevance_config: Any = None,+ scorer: Any = None,+ ccr_config: CCRConfig | None = None,+ ):+ # Hard import — no Python fallback. If the wheel is missing the+ # caller must build it (scripts/build_rust_extension.sh) or+ # install a prebuilt one. Failing loudly is better than silent+ # degradation; see feedback memory `feedback_no_silent_fallbacks.md`.+ from headroom._core import (+ SmartCrusher as _RustSmartCrusher,)-- plan = CompressionPlan(- strategy=analysis.recommended_strategy,- constant_fields=analysis.constant_fields if self.config.factor_out_constants else {},+ from headroom._core import (+ SmartCrusherConfig as _RustSmartCrusherConfig,)- # Handle SKIP - keep all items (shouldn't normally reach here)- if analysis.recommended_strategy == CompressionStrategy.SKIP:- plan.keep_indices = list(range(len(items)))- return plan-- if analysis.recommended_strategy == CompressionStrategy.TIME_SERIES:- plan = self._plan_time_series(- analysis,- items,- plan,- query_context,- preserve_fields,- max_items,- item_strings=item_strings,- )+ cfg = config or SmartCrusherConfig()+ self.config = cfg- elif analysis.recommended_strategy == CompressionStrategy.CLUSTER_SAMPLE:- plan = self._plan_cluster_sample(- analysis,- items,- plan,- query_context,- preserve_fields,- max_items,- item_strings=item_strings,- )+ # CCR config is preserved on `self` for callers that read it+ # back (`headroom.proxy.server` does), but the Rust port doesn't+ # exercise it: Stage 3c.1 keeps CCR marker injection disabled+ # because the Rust port has no compression store. When Stage+ # 3c.2 lands the CCR port, this wires through to Rust.+ if ccr_config is None:+ self._ccr_config = CCRConfig(enabled=True, inject_retrieval_marker=False)+ else:+ self._ccr_config = ccr_config- elif analysis.recommended_strategy == CompressionStrategy.TOP_N:- plan = self._plan_top_n(- analysis,- items,- plan,- query_context,- preserve_fields,- max_items,- item_strings=item_strings,+ # `relevance_config` and `scorer` are accepted for source+ # compatibility but currently dropped — Stage 3c.1 ships with+ # the Rust default `HybridScorer`. Custom scorers re-attach in+ # Stage 3c.2 when the relevance crate gains a Python-bridged+ # constructor surface.+ if relevance_config is not None or scorer is not None:+ logger.debug(+ "SmartCrusher: relevance_config/scorer args are ignored in "+ "Stage 3c.1 (Rust port uses default HybridScorer). They "+ "will be wired through in Stage 3c.2.")- else: # SMART_SAMPLE or NONE- plan = self._plan_smart_sample(- analysis,- items,- plan,- query_context,- preserve_fields,- max_items,- item_strings=item_strings,+ # Build the Rust crusher with every field from the Python+ # config, plus the relevance_threshold default (0.3) — the+ # Python dataclass doesn't carry that field; it lives on+ # `RelevanceScorerConfig` instead.+ self._rust = _RustSmartCrusher(+ _RustSmartCrusherConfig(+ enabled=cfg.enabled,+ min_items_to_analyze=cfg.min_items_to_analyze,+ min_tokens_to_crush=cfg.min_tokens_to_crush,+ variance_threshold=cfg.variance_threshold,+ uniqueness_threshold=cfg.uniqueness_threshold,+ similarity_threshold=cfg.similarity_threshold,+ max_items_after_crush=cfg.max_items_after_crush,+ preserve_change_points=cfg.preserve_change_points,+ factor_out_constants=cfg.factor_out_constants,+ include_summaries=cfg.include_summaries,+ use_feedback_hints=cfg.use_feedback_hints,+ toin_confidence_threshold=cfg.toin_confidence_threshold,+ dedup_identical_items=cfg.dedup_identical_items,+ first_fraction=cfg.first_fraction,+ last_fraction=cfg.last_fraction,+ relevance_threshold=0.3,)-- return plan-- def _plan_time_series(- self,- analysis: ArrayAnalysis,- items: list[dict],- plan: CompressionPlan,- query_context: str = "",- preserve_fields: list[str] | None = None,- max_items: int | None = None,- item_strings: list[str] | None = None,- ) -> CompressionPlan:- """Plan compression for time series data.-- Keeps items around change points (anomalies) plus first/last items.- Uses STATISTICAL outlier detection for important items.- Uses RelevanceScorer for semantic matching of user queries.-- Args:- preserve_fields: TOIN-learned fields that users commonly retrieve.- Items where query_context matches these field values get priority.- max_items: Thread-safe max items limit (defaults to config value).- """- # Use provided max_items or fall back to config- effective_max = max_items if max_items is not None else self.config.max_items_after_crush- n = len(items)- keep_indices = set()-- # 1. Dynamic anchor selection (replaces static first 3 + last 2)- anchor_pattern = self._map_to_anchor_pattern(CompressionStrategy.TIME_SERIES)- anchor_indices = self._anchor_selector.select_anchors(- items=items,- max_items=effective_max,- pattern=anchor_pattern,- query=query_context or None,)- keep_indices.update(anchor_indices)-- # 2. Items around change points from numeric fields- for stats in analysis.field_stats.values():- if stats.change_points:- for cp in stats.change_points:- # Keep a window around each change point- for offset in range(-2, 3):- idx = cp + offset- if 0 <= idx < n:- keep_indices.add(idx)-- # 3. Structural outlier items (STATISTICAL detection - no hardcoded keywords)- outlier_indices = _detect_structural_outliers(items)- keep_indices.update(outlier_indices)-- # 3b. Error items via KEYWORD detection (PRESERVATION GUARANTEE)- # This is critical - errors must ALWAYS be preserved regardless of structure- error_indices = _detect_error_items_for_preservation(items)- keep_indices.update(error_indices)-- # 4. Items matching query anchors (DETERMINISTIC exact match)- # Anchors provide reliable preservation for specific entity lookups (UUIDs, IDs, names)- if query_context:- anchors = extract_query_anchors(query_context)- for i, item in enumerate(items):- if item_matches_anchors(item, anchors):- keep_indices.add(i)-- # 5. Items with high relevance to query context (PROBABILISTIC semantic match)- if query_context:- # Reuse pre-computed item_strings if available- item_strs = (- item_strings- if item_strings is not None- else [json.dumps(item, default=str) for item in items]- )- scores = self._scorer.score_batch(item_strs, query_context)- for i, score in enumerate(scores):- if score.score >= self._relevance_threshold:- keep_indices.add(i)- # 5b. TOIN preserve_fields: boost items where query matches these fields- # Note: preserve_fields are SHA256[:8] hashes, use helper to match- if preserve_fields and query_context:- for i, item in enumerate(items):- if _item_has_preserve_field_match(item, preserve_fields, query_context):- keep_indices.add(i)-- # Limit to effective_max while ALWAYS preserving outliers and anomalies- keep_indices = self._prioritize_indices(keep_indices, items, n, analysis, effective_max)-- plan.keep_indices = sorted(keep_indices)- return plan-- def _plan_cluster_sample(- self,- analysis: ArrayAnalysis,- items: list[dict],- plan: CompressionPlan,- query_context: str = "",- preserve_fields: list[str] | None = None,- max_items: int | None = None,- item_strings: list[str] | None = None,- ) -> CompressionPlan:- """Plan compression for clusterable data (like logs).-- Uses clustering plus STATISTICAL outlier detection.- Uses RelevanceScorer for semantic matching of user queries.+ def crush(self, content: str, query: str = "", bias: float = 1.0) -> CrushResult:+ """Crush a single JSON content string.- Args:- preserve_fields: TOIN-learned fields that users commonly retrieve.- Items where query_context matches these field values get priority.- max_items: Thread-safe max items limit (defaults to config value).+ Mirrors the retired Python method. Returns a `CrushResult`+ dataclass so call sites that destructure with `asdict()` keep+ working."""- # Use provided max_items or fall back to config- effective_max = max_items if max_items is not None else self.config.max_items_after_crush- n = len(items)- keep_indices = set()-- # 1. Dynamic anchor selection (replaces static first 3 + last 2)- anchor_pattern = self._map_to_anchor_pattern(CompressionStrategy.CLUSTER_SAMPLE)- anchor_indices = self._anchor_selector.select_anchors(- items=items,- max_items=effective_max,- pattern=anchor_pattern,- query=query_context or None,+ r = self._rust.crush(content, query, bias)+ return CrushResult(+ compressed=r.compressed,+ original=r.original,+ was_modified=r.was_modified,+ strategy=r.strategy,)- keep_indices.update(anchor_indices)-- # 2. Structural outlier items (STATISTICAL detection - no hardcoded keywords)- outlier_indices = _detect_structural_outliers(items)- keep_indices.update(outlier_indices)-- # 2b. Error items via KEYWORD detection (PRESERVATION GUARANTEE)- # This is critical - errors must ALWAYS be preserved regardless of structure- error_indices = _detect_error_items_for_preservation(items)- keep_indices.update(error_indices)-- # 3. Cluster by message-like field and keep representatives- # Find a high-cardinality string field (likely message field)- message_field = None- max_uniqueness = 0.0- for name, stats in analysis.field_stats.items():- if stats.field_type == "string" and stats.unique_ratio > max_uniqueness:- # Prefer fields with moderate to high uniqueness (message-like)- if stats.unique_ratio > 0.3:- message_field = name- max_uniqueness = stats.unique_ratio-- if message_field:- plan.cluster_field = message_field-- # Simple clustering: group by first 50 chars of message- clusters: dict[str, list[int]] = {}- for i, item in enumerate(items):- msg = str(item.get(message_field, ""))[:50]- msg_hash = hashlib.md5(msg.encode()).hexdigest()[:8] # nosec B324- if msg_hash not in clusters:- clusters[msg_hash] = []- clusters[msg_hash].append(i)-- # Keep 1-2 representatives from each cluster- for indices in clusters.values():- for idx in indices[:2]:- keep_indices.add(idx)-- # 4. Items matching query anchors (DETERMINISTIC exact match)- # Anchors provide reliable preservation for specific entity lookups (UUIDs, IDs, names)- if query_context:- anchors = extract_query_anchors(query_context)- for i, item in enumerate(items):- if item_matches_anchors(item, anchors):- keep_indices.add(i)- # 5. Items with high relevance to query context (PROBABILISTIC semantic match)- if query_context:- # Reuse pre-computed item_strings if available- item_strs = (- item_strings- if item_strings is not None- else [json.dumps(item, default=str) for item in items]- )- scores = self._scorer.score_batch(item_strs, query_context)- for i, score in enumerate(scores):- if score.score >= self._relevance_threshold:- keep_indices.add(i)-- # 5b. TOIN preserve_fields: boost items where query matches these fields- # Note: preserve_fields are SHA256[:8] hashes, use helper to match- if preserve_fields and query_context:- for i, item in enumerate(items):- if _item_has_preserve_field_match(item, preserve_fields, query_context):- keep_indices.add(i)-- # Limit total while ALWAYS preserving outliers and anomalies- keep_indices = self._prioritize_indices(keep_indices, items, n, analysis, effective_max)-- plan.keep_indices = sorted(keep_indices)- return plan-- def _plan_top_n(+ def _smart_crush_content(self,- analysis: ArrayAnalysis,- items: list[dict],- plan: CompressionPlan,+ content: str,query_context: str = "",- preserve_fields: list[str] | None = None,- max_items: int | None = None,- item_strings: list[str] | None = None,- ) -> CompressionPlan:- """Plan compression for scored/ranked data.-- For data with a score/relevance field, that field IS the primary relevance- signal. Our internal relevance scoring is SECONDARY - it's used to find- potential "needle" items that the original scoring might have missed.-- Strategy:- 1. Keep top N by score (the original system's relevance ranking)- 2. Add structural outliers (errors, anomalies)- 3. Add high-confidence relevance matches (needles the user is looking for)+ tool_name: str | None = None,+ bias: float = 1.0,+ ) -> tuple[str, bool, str]:+ """Apply smart crushing; return `(crushed, was_modified, info)`.- Args:- preserve_fields: TOIN-learned fields that users commonly retrieve.- Items where query_context matches these field values get priority.- max_items: Thread-safe max items limit (defaults to config value).+ Mirrors the retired Python method's tuple shape. `tool_name` is+ accepted for API compatibility and currently ignored — Stage+ 3c.1 has no per-tool TOIN learning hook."""- # Use provided max_items or fall back to config- effective_max = max_items if max_items is not None else self.config.max_items_after_crush-- # Find score field using STATISTICAL detection (no hardcoded field names)- score_field = None- max_confidence = 0.0- for name, stats in analysis.field_stats.items():- is_score, confidence = _detect_score_field_statistically(stats, items)- if is_score and confidence > max_confidence:- score_field = name- max_confidence = confidence-- if not score_field:- return self._plan_smart_sample(- analysis,- items,- plan,- query_context,- preserve_fields,- effective_max,- item_strings=item_strings,- )-- plan.sort_field = score_field- keep_indices = set()+ crushed, was_modified, info = self._rust.smart_crush_content(content, query_context, bias)+ return crushed, was_modified, info- # 1. TOP N by score FIRST (the primary relevance signal)- # The original system's score field is the authoritative ranking- scored_items = [(i, item.get(score_field, 0)) for i, item in enumerate(items)]- scored_items.sort(key=lambda x: x[1], reverse=True)-- # Reserve slots for outliers- top_count = max(0, effective_max - 3)- for idx, _ in scored_items[:top_count]:- keep_indices.add(idx)-- # 2. Structural outlier items (STATISTICAL detection - no hardcoded keywords)- outlier_indices = _detect_structural_outliers(items)- keep_indices.update(outlier_indices)+ def _extract_context_from_messages(self, messages: list[dict[str, Any]]) -> str:+ """Build a query string from the last 5 user messages + recent+ assistant tool-call arguments. Used by `apply()` to derive the+ relevance context per-request.- # 2b. Error items via KEYWORD detection (PRESERVATION GUARANTEE)- # This is critical - errors must ALWAYS be preserved regardless of structure- error_indices = _detect_error_items_for_preservation(items)- keep_indices.update(error_indices)+ Pure Python because it walks the message envelope, not the+ compressed payload. The retired implementation lived inline on+ `SmartCrusher`; preserved here unchanged.+ """+ context_parts: list[str] = []+ user_message_count = 0- # 3. Items matching query anchors (DETERMINISTIC exact match) - ADDITIVE- # Anchors provide reliable preservation for specific entity lookups (UUIDs, IDs, names)- # These are ALWAYS preserved since they represent explicit user intent- if query_context:- anchors = extract_query_anchors(query_context)- for i, item in enumerate(items):- if i not in keep_indices and item_matches_anchors(item, anchors):- keep_indices.add(i)+ for msg in reversed(messages):+ if msg.get("role") == "user":+ content = msg.get("content")+ if isinstance(content, str):+ context_parts.append(content)+ elif isinstance(content, list):+ for block in content:+ if isinstance(block, dict) and block.get("type") == "text":+ text = block.get("text", "")+ if text:+ context_parts.append(text)- # 4. HIGH-CONFIDENCE relevance matches (potential needles) - ADDITIVE only- # Only add items that are NOT already in top N but match the query strongly- # Use a higher threshold (0.5) since the score field already captures relevance- if query_context:- # Reuse pre-computed item_strings if available- item_strs = (- item_strings- if item_strings is not None- else [json.dumps(item, default=str) for item in items]- )- scores = self._scorer.score_batch(item_strs, query_context)- # Higher threshold and limit count to avoid adding everything- high_threshold = max(0.5, self._relevance_threshold * 2)- added_count = 0- max_relevance_adds = 3 # Limit additional relevance matches- for i, score in enumerate(scores):- if i not in keep_indices and score.score >= high_threshold:- keep_indices.add(i)- added_count += 1- if added_count >= max_relevance_adds:- break+ user_message_count += 1+ if user_message_count >= 5:+ break- # 4b. TOIN preserve_fields: boost items where query matches these fields- # Note: preserve_fields are SHA256[:8] hashes, use helper to match- if preserve_fields and query_context:- for i, item in enumerate(items):- if i not in keep_indices: # Only add if not already kept- if _item_has_preserve_field_match(item, preserve_fields, query_context):- keep_indices.add(i)+ if msg.get("role") == "assistant" and msg.get("tool_calls"):+ for tc in msg.get("tool_calls", []):+ if isinstance(tc, dict):+ func = tc.get("function", {})+ args = func.get("arguments", "")+ if isinstance(args, str) and args:+ context_parts.append(args)- plan.keep_count = len(keep_indices)- plan.keep_indices = sorted(keep_indices)- return plan+ return " ".join(context_parts)- def _plan_smart_sample(+ def apply(self,- analysis: ArrayAnalysis,- items: list[dict],- plan: CompressionPlan,- query_context: str = "",- preserve_fields: list[str] | None = None,- max_items: int | None = None,- item_strings: list[str] | None = None,- ) -> CompressionPlan:- """Plan smart statistical sampling using STATISTICAL detection.-- Always keeps:- - Dynamic anchor positions (based on data pattern and query context)- - Structural outliers (items with rare fields or rare status values)- - Anomalous numeric items (> 2 std from mean)- - Items around change points- - Items with high relevance to query context (via RelevanceScorer)-- Uses STATISTICAL detection instead of hardcoded keywords.+ messages: list[dict[str, Any]],+ tokenizer: Tokenizer,+ **kwargs: Any,+ ) -> TransformResult:+ """Transform-protocol entry point. Walks every tool/tool_result+ message, applies SmartCrusher to large enough payloads, and+ replaces the message content with `<crushed>\\n<digest_marker>`.- Args:- preserve_fields: TOIN-learned fields that users commonly retrieve.- Items where query_context matches these field values get priority.- max_items: Thread-safe max items limit (defaults to config value).+ Pure orchestration — the per-message compression delegates to+ Rust via `_smart_crush_content`."""- # Use provided max_items or fall back to config- effective_max = max_items if max_items is not None else self.config.max_items_after_crush-- n = len(items)- keep_indices = set()-- # 1. Dynamic anchor selection (replaces static first 3 + last 2)- anchor_pattern = self._map_to_anchor_pattern(CompressionStrategy.SMART_SAMPLE)- anchor_indices = self._anchor_selector.select_anchors(- items=items,- max_items=effective_max,- pattern=anchor_pattern,- query=query_context or None,- )- keep_indices.update(anchor_indices)-- # 2. Structural outlier items (STATISTICAL detection - no hardcoded keywords)- outlier_indices = _detect_structural_outliers(items)- keep_indices.update(outlier_indices)-- # 2b. Error items via KEYWORD detection (PRESERVATION GUARANTEE)- # This is critical - errors must ALWAYS be preserved regardless of structure- error_indices = _detect_error_items_for_preservation(items)- keep_indices.update(error_indices)-- # 3. Anomalous numeric items (> 2 std from mean)- for name, stats in analysis.field_stats.items():- if stats.field_type == "numeric" and stats.mean_val is not None and stats.variance:- std = stats.variance**0.5- if std > 0:- threshold = self.config.variance_threshold * std- for i, item in enumerate(items):- val = item.get(name)- if isinstance(val, int | float):- if abs(val - stats.mean_val) > threshold:- keep_indices.add(i)-- # 4. Items around change points (if detected)- if self.config.preserve_change_points:- for stats in analysis.field_stats.values():- if stats.change_points:- for cp in stats.change_points:- # Keep items around change point- for offset in range(-1, 2):- idx = cp + offset- if 0 <= idx < n:- keep_indices.add(idx)+ tokens_before = tokenizer.count_messages(messages)+ result_messages = deep_copy_messages(messages)+ transforms_applied: list[str] = []+ markers_inserted: list[str] = []+ warnings: list[str] = []- # 5. Items matching query anchors (DETERMINISTIC exact match)- # Anchors provide reliable preservation for specific entity lookups (UUIDs, IDs, names)- if query_context:- anchors = extract_query_anchors(query_context)- for i, item in enumerate(items):- if item_matches_anchors(item, anchors):- keep_indices.add(i)+ query_context = self._extract_context_from_messages(result_messages)+ crushed_count = 0+ frozen_message_count = kwargs.get("frozen_message_count", 0)- # 6. Items with high relevance to query context (PROBABILISTIC semantic match)- if query_context:- # Reuse pre-computed item_strings if available- item_strs = (- item_strings- if item_strings is not None- else [json.dumps(item, default=str) for item in items]- )- scores = self._scorer.score_batch(item_strs, query_context)- for i, score in enumerate(scores):- if score.score >= self._relevance_threshold:- keep_indices.add(i)+ for msg_idx, msg in enumerate(result_messages):+ if msg_idx < frozen_message_count:+ continue- # 6b. TOIN preserve_fields: boost items where query matches these fields- # Note: preserve_fields are SHA256[:8] hashes, use helper to match- if preserve_fields and query_context:- for i, item in enumerate(items):- if _item_has_preserve_field_match(item, preserve_fields, query_context):- keep_indices.add(i)+ # OpenAI-style: top-level role=tool with string content.+ if msg.get("role") == "tool":+ content = msg.get("content", "")+ if isinstance(content, str):+ tokens = tokenizer.count_text(content)+ if tokens > self.config.min_tokens_to_crush:+ crushed, was_modified, info = self._smart_crush_content(+ content, query_context+ )+ if was_modified:+ marker = create_tool_digest_marker(compute_short_hash(content))+ msg["content"] = crushed + "\n" + marker+ crushed_count += 1+ markers_inserted.append(marker)+ if info:+ transforms_applied.append(f"smart:{info}")++ # Anthropic-style: content is a list of blocks; each tool_result+ # block has a string content field of its own.+ content = msg.get("content")+ if isinstance(content, list):+ for i, block in enumerate(content):+ if not isinstance(block, dict) or block.get("type") != "tool_result":+ continue+ tool_content = block.get("content", "")+ if not isinstance(tool_content, str):+ continue+ tokens = tokenizer.count_text(tool_content)+ if tokens <= self.config.min_tokens_to_crush:+ continue- # Limit to effective_max while ALWAYS preserving outliers and anomalies- keep_indices = self._prioritize_indices(keep_indices, items, n, analysis, effective_max)+ crushed, was_modified, info = self._smart_crush_content(+ tool_content, query_context+ )+ if was_modified:+ marker = create_tool_digest_marker(compute_short_hash(tool_content))+ content[i]["content"] = crushed + "\n" + marker+ crushed_count += 1+ markers_inserted.append(marker)+ if info:+ transforms_applied.append(f"smart:{info}")- plan.keep_indices = sorted(keep_indices)- return plan+ if crushed_count > 0:+ transforms_applied.insert(0, f"smart_crush:{crushed_count}")- def _execute_plan(- self, plan: CompressionPlan, items: list[dict], analysis: ArrayAnalysis- ) -> list:- """Execute a compression plan and return crushed array.+ tokens_after = tokenizer.count_messages(result_messages)- SCHEMA-PRESERVING: Returns only items from the original array.- No wrappers, no generated text, no metadata keys.- """- result = []+ return TransformResult(+ messages=result_messages,+ tokens_before=tokens_before,+ tokens_after=tokens_after,+ transforms_applied=transforms_applied,+ markers_inserted=markers_inserted,+ warnings=warnings,+ )- # Return only the kept items, preserving original schema- for idx in sorted(plan.keep_indices):- if 0 <= idx < len(items):- # Copy item unchanged - no modifications to schema- result.append(items[idx].copy())- return result+# ─── Convenience function ─────────────────────────────────────────────────def smart_crush_tool_output(@@ -3719,32 +331,12 @@ def smart_crush_tool_output(config: SmartCrusherConfig | None = None,ccr_config: CCRConfig | None = None,) -> tuple[str, bool, str]:- """- Convenience function to smart-crush a single tool output.-- NOTE: CCR markers are DISABLED by default in this convenience function- to maintain backward compatibility (output remains valid JSON).- To enable CCR markers, pass a CCRConfig with inject_retrieval_marker=True.+ """Compress a single tool output. Returns `(crushed, was_modified, info)`.- Args:- content: The tool output content (JSON string).- config: Optional SmartCrusher configuration.- ccr_config: Optional CCR (Compress-Cache-Retrieve) configuration.- By default, CCR is enabled (caching) but markers are disabled.-- Returns:- Tuple of (crushed_content, was_modified, analysis_info).+ Convenience wrapper that builds a one-shot `SmartCrusher` per call.+ `ccr_config` is accepted for source compatibility but currently+ not wired through (Stage 3c.1 keeps CCR marker injection+ disabled — Rust port has no compression store)."""- cfg = config or SmartCrusherConfig()-- # Default: CCR enabled for caching, but markers disabled for clean JSON output- if ccr_config is None:- ccr_cfg = CCRConfig(- enabled=True, # Still cache for retrieval- inject_retrieval_marker=False, # Don't break JSON output- )- else:- ccr_cfg = ccr_config-- crusher = SmartCrusher(cfg, ccr_config=ccr_cfg)+ crusher = SmartCrusher(config=config, ccr_config=ccr_config)return crusher._smart_crush_content(content)diff --git a/tests/test_acceptance.py b/tests/test_acceptance.pyindex 6a4d3e680..c8636c084 100644--- a/tests/test_acceptance.py+++ b/tests/test_acceptance.py@@ -447,11 +447,7 @@ class TestQueryAnchorExtraction:"""If user asks for 'Alice', item with Alice should be preserved."""import json- from headroom.transforms.smart_crusher import (- SmartCrusher,- SmartCrusherConfig,- extract_query_anchors,- )+ from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig# User is searching for 'Alice'messages = [@@ -478,38 +474,31 @@ class TestQueryAnchorExtraction:},]- # Verify anchor extraction works- anchors = extract_query_anchors("Find the user named 'Alice' in the system.")- assert "alice" in anchors-- # Verify crushing preserves Alice+ # End-to-end behavior: the relevance scorer (HybridScorer in+ # the Rust port — BM25 + embedding) should pick up "Alice"+ # from the user message and preserve the matching tool item+ # even though it sits at index 50.config = SmartCrusherConfig(enabled=True,min_items_to_analyze=5,min_tokens_to_crush=100,- max_items_after_crush=10, # Should normally drop Alice at index 50+ max_items_after_crush=10,)crusher = SmartCrusher(config)tokenizer = get_tokenizer()result = crusher.apply(messages, tokenizer)- # Find the crushed tool outputtool_msg = next(m for m in result.messages if m.get("role") == "tool")crushed_content = tool_msg["content"]- # Alice should be preserved even though she's at index 50assert "Alice" in crushed_contentdef test_preserves_needle_by_uuid(self):"""If user asks for a UUID, item with that UUID should be preserved."""import json- from headroom.transforms.smart_crusher import (- SmartCrusher,- SmartCrusherConfig,- extract_query_anchors,- )+ from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfigtarget_uuid = "550e8400-e29b-41d4-a716-446655440000"@@ -537,10 +526,6 @@ class TestQueryAnchorExtraction:},]- # Verify anchor extraction- anchors = extract_query_anchors(f"Get details for request {target_uuid}")- assert target_uuid.lower() in anchors-config = SmartCrusherConfig(enabled=True,min_items_to_analyze=5,@@ -555,7 +540,6 @@ class TestQueryAnchorExtraction:tool_msg = next(m for m in result.messages if m.get("role") == "tool")crushed_content = tool_msg["content"]- # UUID should be preservedassert target_uuid in crushed_contentdiff --git a/tests/test_ccr.py b/tests/test_ccr.pyindex d11454a0c..c68880725 100644--- a/tests/test_ccr.py+++ b/tests/test_ccr.py@@ -235,160 +235,6 @@ class TestCompressionStore:assert store.exists(h)-class TestSmartCrusherCCRIntegration:- """Test SmartCrusher integration with CCR."""-- @pytest.fixture(autouse=True)- def reset_store(self):- """Reset global store before each test."""- reset_compression_store()- yield- reset_compression_store()-- def test_compression_caches_original(self):- """SmartCrusher caches original content when compressing."""- items = [{"id": i, "score": 100 - i, "data": f"item_{i}"} for i in range(100)]- content = json.dumps(items)-- config = SmartCrusherConfig(max_items_after_crush=15)- ccr_config = CCRConfig(- enabled=True,- inject_retrieval_marker=False, # Don't add marker for this test- min_items_to_cache=10,- )-- compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config)-- assert was_modified-- # Check that content was cached- store = get_compression_store()- stats = store.get_stats()- assert stats["entry_count"] >= 1-- def test_retrieval_marker_injected(self):- """CCR marker is injected when configured."""- items = [{"id": i, "score": 100 - i, "data": f"item_{i}"} for i in range(100)]- content = json.dumps(items)-- config = SmartCrusherConfig(max_items_after_crush=15)- ccr_config = CCRConfig(- enabled=True,- inject_retrieval_marker=True,- min_items_to_cache=10,- )-- compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config)-- assert was_modified- # Marker should be present- assert "items compressed" in compressed_str or "hash=" in compressed_str-- def test_small_arrays_not_cached(self):- """Arrays smaller than min_items_to_cache are not cached."""- items = [{"id": i} for i in range(15)]- content = json.dumps(items)-- config = SmartCrusherConfig(max_items_after_crush=10)- ccr_config = CCRConfig(- enabled=True,- min_items_to_cache=50, # Require 50+ items- )-- smart_crush_tool_output(content, config, ccr_config)-- store = get_compression_store()- stats = store.get_stats()- # Should not cache because original has < 50 items- assert stats["entry_count"] == 0-- def test_uncrushed_data_not_cached(self):- """Data that doesn't get crushed is not cached."""- # DB results with unique IDs - shouldn't be crushed- items = [{"id": i, "name": f"User {i}", "email": f"user{i}@test.com"} for i in range(30)]- content = json.dumps(items)-- config = SmartCrusherConfig(max_items_after_crush=10)- ccr_config = CCRConfig(enabled=True, min_items_to_cache=10)-- compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config)-- # If not modified, shouldn't be cached- if not was_modified:- store = get_compression_store()- stats = store.get_stats()- assert stats["entry_count"] == 0-- def test_can_retrieve_after_compression(self):- """Can retrieve original content after compression."""- items = [- {"id": i, "score": 100 - i, "content": f"Document about topic {i}"} for i in range(100)- ]- content = json.dumps(items)-- config = SmartCrusherConfig(max_items_after_crush=15)- ccr_config = CCRConfig(- enabled=True,- inject_retrieval_marker=True,- min_items_to_cache=10,- )-- compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config)-- assert was_modified-- # Extract hash from marker- # Marker format: [100 items compressed to 15. Retrieve more: hash=abc123...]- import re-- match = re.search(r"hash=([a-f0-9]+)", compressed_str)- assert match is not None, f"No hash found in: {compressed_str}"-- hash_key = match.group(1)-- # Retrieve original- store = get_compression_store()- entry = store.retrieve(hash_key)-- assert entry is not None- original_items = json.loads(entry.original_content)- assert len(original_items) == 100-- def test_search_after_compression(self):- """Can search within original content after compression."""- items = [- {"id": 1, "content": "Authentication error: invalid token"},- {"id": 2, "content": "Database connection successful"},- {"id": 3, "content": "User login completed"},- ] + [{"id": i, "content": f"Generic log entry {i}"} for i in range(4, 104)]- content = json.dumps(items)-- config = SmartCrusherConfig(max_items_after_crush=15)- ccr_config = CCRConfig(- enabled=True,- inject_retrieval_marker=True,- min_items_to_cache=10,- )-- compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config)-- assert was_modified-- # Extract hash- import re-- match = re.search(r"hash=([a-f0-9]+)", compressed_str)- hash_key = match.group(1)-- # Search for authentication items- store = get_compression_store()- results = store.search(hash_key, "authentication error token")-- # Should find the authentication error item- assert len(results) >= 1- assert any("Authentication" in r.get("content", "") for r in results)--class TestCCRConfig:"""Test CCR configuration options."""@@ -402,24 +248,6 @@ class TestCCRConfig:assert config.feedback_enabled is Trueassert config.min_items_to_cache == 20- def test_custom_marker_template(self):- """Custom marker template is used."""- items = [{"id": i, "score": 100 - i} for i in range(100)]- content = json.dumps(items)-- config = SmartCrusherConfig(max_items_after_crush=15)- ccr_config = CCRConfig(- enabled=True,- inject_retrieval_marker=True,- min_items_to_cache=10,- marker_template="\n[CUSTOM: {original_count} -> {compressed_count}, key={hash}]",- )-- compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config)-- if was_modified:- assert "CUSTOM:" in compressed_str or "key=" in compressed_str-class TestCCRFeedbackLoop:"""Test CCR feedback tracking for learning."""diff --git a/tests/test_critical_fixes.py b/tests/test_critical_fixes.pyindex fd6d8263f..c00a7d0e5 100644--- a/tests/test_critical_fixes.py+++ b/tests/test_critical_fixes.py@@ -11,8 +11,6 @@ These tests verify the before/after behavior of critical bug fixes:import timefrom unittest.mock import patch-import pytest-class TestTOINConfidenceMathFix:"""Test for CRITICAL: Confidence calculation math error in toin.py:721.@@ -303,146 +301,5 @@ class TestUnboundedStrategyDicts:)-class TestSmartCrusherTOINIntegration:- """Test for CRITICAL: SmartCrusher not calling toin.record_compression().-- BUG: SmartCrusher calls feedback.record_compression() but never calls- toin.record_compression(). This means TOIN only learns from retrieval events,- not from compression events - breaking the feedback loop.-- FIX: Add toin.record_compression() call after compression in SmartCrusher.- """-- def test_smart_crusher_records_to_toin(self):- """SmartCrusher should record compression events to TOIN."""- from headroom.telemetry.models import ToolSignature- from headroom.telemetry.toin import get_toin, reset_toin- from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig-- reset_toin()-- config = SmartCrusherConfig(- min_items_to_analyze=5,- max_items_after_crush=10,- use_feedback_hints=True,- )- crusher = SmartCrusher(config)-- # Create test items that look like search results with a clear score field- # This pattern is crushable because:- # 1. Has a clear numeric score field in BOUNDED range [0,1]- # 2. Has repeated structure with some constant fields (type, language)- # 3. Score values vary within the bounded range- items = [- {- "name": f"repo_{i}",- "relevance_score": (50 - i) / 50.0, # Bounded [0,1] - descending order- "type": "repository", # Constant field- "language": "python" if i % 3 == 0 else "javascript", # Low cardinality- "description": f"Description {i % 5}", # Low cardinality- }- for i in range(50)- ]-- # Get TOIN instance and check initial state- toin = get_toin()- len(toin._patterns)-- # Crush the array- result, info, markers, _summary = crusher._crush_array(- items, query_context="test query", tool_name="test_tool"- )-- # Verify compression happened (not skipped)- assert "skip" not in info.lower(), (- f"Compression was skipped: {info}. Test needs crushable data."- )-- # Get the signature that would have been created- sig = ToolSignature.from_items(items)-- # Check TOIN was notified- with toin._lock:- pattern = toin._patterns.get(sig.structure_hash)-- # After fix, TOIN should have a pattern for this tool's signature- assert pattern is not None, (- f"TOIN should have recorded the compression event. "- f"Info: {info}, pattern count: {len(toin._patterns)}"- )- if pattern:- assert pattern.total_compressions >= 1, (- f"Pattern should have at least 1 compression recorded, got {pattern.total_compressions}"- )--class TestAllFixesIntegrated:"""Integration tests ensuring all fixes work together."""-- def test_full_feedback_loop(self):- """Test complete feedback loop: compress -> store -> retrieve -> learn."""- from headroom.cache.compression_feedback import (- reset_compression_feedback,- )- from headroom.cache.compression_store import reset_compression_store- from headroom.telemetry.models import ToolSignature- from headroom.telemetry.toin import get_toin, reset_toin- from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig-- # Reset all singletons- reset_toin()- reset_compression_store()- reset_compression_feedback()-- # Setup- config = SmartCrusherConfig(- min_items_to_analyze=5,- max_items_after_crush=10,- use_feedback_hints=True,- )- crusher = SmartCrusher(config)-- # Create test items that look like API responses with scoring- # This pattern is crushable because:- # 1. Has a clear numeric score field in BOUNDED range [0,1]- # 2. Has constant fields (status, type)- # 3. Has enough items for compression (100)- items = [- {- "priority": (100 - i) / 100.0, # Bounded [0,1] - descending order- "status": "ok", # Constant field- "type": "response", # Constant field- "data": f"content_{i % 10}", # Low cardinality (only 10 unique values)- }- for i in range(100)- ]-- # Step 1: Compress- result, info, markers, _summary = crusher._crush_array(- items, query_context="find status", tool_name="api_response"- )-- # Verify compression happened (not skipped)- assert "skip" not in info.lower(), (- f"Compression was skipped: {info}. Test needs crushable data."- )-- # Step 2: Check TOIN was notified (after fix)- toin = get_toin()- sig = ToolSignature.from_items(items)-- with toin._lock:- toin_pattern = toin._patterns.get(sig.structure_hash)-- # After fix, TOIN should have the pattern- assert toin_pattern is not None, (- f"TOIN should have learned from the compression event. Info: {info}"- )- assert toin_pattern.total_compressions >= 1, (- f"TOIN pattern should have recorded compression, got {toin_pattern.total_compressions}"- )---# Run specific test to verify fix-if __name__ == "__main__":- pytest.main([__file__, "-v", "--tb=short"])diff --git a/tests/test_crushability.py b/tests/test_crushability.pydeleted file mode 100644index e67ac428b..000000000--- a/tests/test_crushability.py+++ /dev/null@@ -1,423 +0,0 @@-"""Tests for SmartCrusher crushability analysis.--These tests verify that SmartCrusher correctly identifies when it's SAFE-to crush data vs when it should SKIP crushing.--The key insight: High variability + No importance signal = DON'T CRUSH.--Test scenarios:-1. DB results (unique entities, no signal) → SKIP-2. Search results (has score field) → CRUSH using score-3. Log entries (has errors) → CRUSH keeping errors-4. Time series (has anomalies) → CRUSH keeping anomalies-5. Repetitive data (low uniqueness) → CRUSH with sampling-"""--import json--import pytest--from headroom.transforms.smart_crusher import (- CompressionStrategy,- SmartAnalyzer,- SmartCrusherConfig,- smart_crush_tool_output,-)---class TestCrushabilityDetection:- """Test the crushability analysis logic."""-- @pytest.fixture- def analyzer(self):- """Create a SmartAnalyzer instance."""- return SmartAnalyzer(SmartCrusherConfig())-- def test_db_results_not_crushable(self, analyzer):- """DB query results with unique IDs and no signal should NOT be crushed."""- # Simulate: SELECT * FROM users LIMIT 50- items = [- {- "id": i,- "name": f"User {i}",- "email": f"user{i}@example.com",- "department": "Engineering",- }- for i in range(50)- ]-- analysis = analyzer.analyze_array(items)-- # Should detect unique entities with no importance signal- assert analysis.crushability is not None- assert not analysis.crushability.crushable, (- f"DB results should NOT be crushable. "- f"Reason: {analysis.crushability.reason}, "- f"Signals: {analysis.crushability.signals_present}"- )- assert analysis.recommended_strategy == CompressionStrategy.SKIP- assert "unique" in analysis.crushability.reason.lower()-- def test_db_results_with_unique_uuid(self, analyzer):- """DB results with UUID field should NOT be crushed."""- items = [- {- "uuid": f"550e8400-e29b-41d4-a716-44665544{i:04d}",- "name": f"Record {i}",- "value": i * 10,- }- for i in range(50)- ]-- analysis = analyzer.analyze_array(items)-- assert analysis.crushability is not None- assert not analysis.crushability.crushable- assert analysis.crushability.has_id_field-- def test_search_results_crushable(self, analyzer):- """Search results with score field SHOULD be crushed."""- items = [- {- "id": i,- "title": f"Document {i}",- "snippet": f"This is document {i} content...",- "score": 1.0 - (i * 0.01), # Decreasing relevance- }- for i in range(100)- ]-- analysis = analyzer.analyze_array(items)-- # Should detect score field as importance signal- assert analysis.crushability is not None- assert analysis.crushability.crushable, (- f"Search results should be crushable. Reason: {analysis.crushability.reason}"- )- assert analysis.crushability.has_score_field- assert any("score" in s for s in analysis.crushability.signals_present)-- def test_log_entries_with_errors_crushable(self, analyzer):- """Log entries containing structural outliers SHOULD be crushed (outliers preserved)."""- items = []- for i in range(100):- item = {- "id": i,- "timestamp": f"2024-01-15T10:{i:02d}:00Z",- "message": f"Request processed successfully - {i}",- "level": "INFO",- }- # Add some errors - these are STRUCTURAL OUTLIERS (have extra "error" field)- if i % 20 == 0:- item["level"] = "ERROR"- item["message"] = f"Connection failed: timeout at {i}"- item["error"] = "TimeoutError" # Extra field that most items don't have- items.append(item)-- analysis = analyzer.analyze_array(items)-- # Should detect structural outliers (items with rare fields like "error")- assert analysis.crushability is not None- assert analysis.crushability.crushable- # Now uses structural_outliers instead of keyword-based error count- assert any(- "structural_outliers" in s or "outlier" in s.lower()- for s in analysis.crushability.signals_present- )-- def test_time_series_with_anomalies_crushable(self, analyzer):- """Time series with numeric anomalies SHOULD be crushed."""- items = []- for i in range(100):- value = 100.0 # Normal value- if i in [25, 50, 75]: # Anomaly points- value = 999.0- items.append(- {- "id": i,- "timestamp": i,- "cpu_usage": value,- }- )-- analysis = analyzer.analyze_array(items)-- # Should detect anomalies as importance signal- assert analysis.crushability is not None- assert analysis.crushability.crushable- assert analysis.crushability.anomaly_count > 0-- def test_repetitive_data_crushable(self, analyzer):- """Repetitive data (low uniqueness) SHOULD be crushable."""- # Same status repeated many times- items = [- {- "id": i,- "status": "success", # Same for all- "code": 200, # Same for all- "message": "OK", # Same for all- }- for i in range(100)- ]-- analysis = analyzer.analyze_array(items)-- # Should detect low uniqueness - safe to sample- assert analysis.crushability is not None- assert analysis.crushability.crushable- # Can be "low_uniqueness" or "repetitive_content_with_ids"- assert (- "low_uniqueness" in analysis.crushability.reason- or "repetitive" in analysis.crushability.reason- )-- def test_file_listing_not_crushable(self, analyzer):- """File listing with unique paths should NOT be crushed."""- items = [- {- "id": i,- "path": f"/home/user/project/src/module{i}/file{i}.py",- "size": 1000 + i,- "modified": f"2024-01-{(i % 28) + 1:02d}",- }- for i in range(50)- ]-- analysis = analyzer.analyze_array(items)-- # Paths are highly unique, no importance signal- assert analysis.crushability is not None- # Should NOT crush file listings- assert not analysis.crushability.crushable or analysis.crushability.confidence < 0.7-- def test_order_list_not_crushable(self, analyzer):- """Order list with unique order IDs should NOT be crushed."""- items = [- {- "order_id": f"ORD-2024-{i:05d}",- "customer": f"Customer {i}",- "total": 50.0 + i,- "status": "completed",- }- for i in range(50)- ]-- analysis = analyzer.analyze_array(items)-- # Each order is a unique entity- assert analysis.crushability is not None- # order_id contains 'id' pattern- assert not analysis.crushability.crushable---class TestCrushabilityEndToEnd:- """End-to-end tests for crushability-aware crushing."""-- def test_db_results_preserved_completely(self):- """DB results should be returned unchanged when not crushable."""- items = [{"id": i, "name": f"User {i}", "email": f"user{i}@test.com"} for i in range(30)]- content = json.dumps(items)-- config = SmartCrusherConfig(max_items_after_crush=10)- crushed, was_modified, info = smart_crush_tool_output(content, config)-- # Should NOT be modified (skip crushing)- if was_modified:- result = json.loads(crushed)- # If it was modified, all items should still be there- assert len(result) == 30, (- f"DB results should not lose items! Had 30, got {len(result)}. Info: {info}"- )-- def test_search_results_crushed_by_score(self):- """Search results should be crushed using score field."""- items = [- {- "id": i,- "title": f"Result {i}",- "score": 100 - i, # Higher score = more relevant- }- for i in range(100)- ]- content = json.dumps(items)-- config = SmartCrusherConfig(max_items_after_crush=15)- crushed, was_modified, info = smart_crush_tool_output(content, config)-- assert was_modified- result = json.loads(crushed)- assert len(result) < 100-- # Top scores should be preserved- scores = [item.get("score", 0) for item in result]- assert max(scores) >= 90 # Top items preserved-- def test_mixed_data_with_errors_preserves_errors(self):- """Data with errors should crush but preserve ALL errors."""- items = []- error_ids = [5, 25, 45, 65, 85]- for i in range(100):- item = {"id": i, "data": f"value_{i}"}- if i in error_ids:- item["status"] = "failed"- item["error"] = f"Error at {i}"- items.append(item)-- content = json.dumps(items)- config = SmartCrusherConfig(max_items_after_crush=20)- crushed, was_modified, info = smart_crush_tool_output(content, config)-- result = json.loads(crushed)-- # All errors must be preserved- error_count = sum(1 for item in result if item.get("error"))- assert error_count == len(error_ids), (- f"All {len(error_ids)} errors should be preserved, got {error_count}"- )---class TestCrushabilitySignals:- """Test individual signal detection."""-- @pytest.fixture- def analyzer(self):- return SmartAnalyzer(SmartCrusherConfig())-- def test_detects_id_field_variations(self, analyzer):- """Should detect various ID field naming patterns."""- test_cases = [- ("id", [{"id": i} for i in range(20)]),- ("uuid", [{"uuid": f"uuid-{i}"} for i in range(20)]),- ("_id", [{"_id": f"mongo-{i}"} for i in range(20)]),- ("pk", [{"pk": i} for i in range(20)]),- ("key", [{"key": f"key-{i}"} for i in range(20)]),- ("user_id", [{"user_id": i} for i in range(20)]),- ]-- for field_name, items in test_cases:- analysis = analyzer.analyze_array(items)- assert analysis.crushability is not None- assert analysis.crushability.has_id_field, f"Should detect '{field_name}' as ID field"-- def test_detects_score_field_variations(self, analyzer):- """Should detect various score field naming patterns."""- test_cases = [- "score",- "rank",- "relevance",- "confidence",- "_score",- "rating",- ]-- for field_name in test_cases:- items = [{field_name: i * 0.1, "data": f"item_{i}"} for i in range(20)]- analysis = analyzer.analyze_array(items)- assert analysis.crushability is not None- assert analysis.crushability.has_score_field, (- f"Should detect '{field_name}' as score field"- )-- def test_detects_error_keywords(self, analyzer):- """Should detect various error keyword patterns."""- error_keywords = ["error", "exception", "failed", "failure", "critical", "fatal"]-- for keyword in error_keywords:- items = [{"id": i, "msg": "OK"} for i in range(20)]- items[10]["msg"] = f"Something {keyword} happened"-- analysis = analyzer.analyze_array(items)- assert analysis.crushability is not None- assert analysis.crushability.error_item_count >= 1, (- f"Should detect '{keyword}' as error indicator"- )---class TestCrushabilityEdgeCases:- """Test edge cases in crushability analysis."""-- @pytest.fixture- def analyzer(self):- return SmartAnalyzer(SmartCrusherConfig())-- def test_empty_array(self, analyzer):- """Empty array should not crash."""- analysis = analyzer.analyze_array([])- assert analysis.recommended_strategy == CompressionStrategy.NONE-- def test_small_array_skipped(self, analyzer):- """Arrays below min_items_to_analyze should be skipped."""- items = [{"id": i} for i in range(3)]- analysis = analyzer.analyze_array(items)- assert analysis.recommended_strategy == CompressionStrategy.NONE-- def test_mixed_signals(self, analyzer):- """Data with multiple signals should still be crushable."""- items = []- for i in range(100):- item = {- "id": i,- "score": 100 - i, # Score signal- "value": 50.0,- }- if i == 50:- item["error"] = "Test error" # Error signal- item["value"] = 999.0 # Anomaly signal- items.append(item)-- analysis = analyzer.analyze_array(items)- assert analysis.crushability is not None- assert analysis.crushability.crushable- assert len(analysis.crushability.signals_present) >= 2-- def test_all_items_are_errors(self, analyzer):- """When all items are errors, keyword detection finds them as a signal.-- With keyword-based error detection (for the preservation guarantee),- when ALL items have error keywords, we detect error_keywords:50 as a- signal. This makes the data technically crushable.-- However, since ALL items are errors, they will ALL be preserved due to- the preservation guarantee. The end result is the same - no data loss.- """- items = [{"id": i, "error": f"Error {i}", "status": "failed"} for i in range(50)]-- analysis = analyzer.analyze_array(items)- assert analysis.crushability is not None-- # With keyword-based error detection, all 50 items contain error keywords- # This IS a signal (error_keywords:50), making the data crushable.- # However, all 50 items will be preserved due to the preservation guarantee.- assert analysis.crushability.crushable- assert "error_keywords:50" in analysis.crushability.signals_present---class TestCrushabilityConfidence:- """Test confidence scoring in crushability analysis."""-- @pytest.fixture- def analyzer(self):- return SmartAnalyzer(SmartCrusherConfig())-- def test_high_confidence_for_clear_cases(self, analyzer):- """Clear-cut cases should have high confidence."""- # Low uniqueness - clearly safe- items = [{"status": "ok", "code": 200} for _ in range(100)]- analysis = analyzer.analyze_array(items)- assert analysis.crushability is not None- assert analysis.crushability.confidence >= 0.8-- def test_lower_confidence_for_ambiguous_cases(self, analyzer):- """Ambiguous cases should have lower confidence."""- # Medium uniqueness with weak signal- items = [- {"id": i, "value": i % 10, "status": "active" if i % 2 == 0 else "inactive"}- for i in range(100)- ]- # Add one error to provide weak signal- items[50]["error"] = "minor issue"-- analysis = analyzer.analyze_array(items)- assert analysis.crushability is not None- # Should be lower confidence due to ambiguity- assert analysis.crushability.confidence <= 0.7diff --git a/tests/test_relevance.py b/tests/test_relevance.pyindex bdfa4de1a..6da26e1a0 100644--- a/tests/test_relevance.py+++ b/tests/test_relevance.py@@ -273,33 +273,6 @@ class TestEmbeddingAvailable:class TestSmartCrusherIntegration:"""Integration tests for SmartCrusher with RelevanceScorer."""- def test_crusher_uses_scorer(self):- """SmartCrusher uses relevance scorer for context matching."""- from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig-- config = SmartCrusherConfig(- min_items_to_analyze=3,- min_tokens_to_crush=0,- max_items_after_crush=5,- )- crusher = SmartCrusher(config=config)-- # Verify scorer is initialized- assert hasattr(crusher, "_scorer")- assert hasattr(crusher, "_relevance_threshold")-- def test_custom_scorer_injection(self):- """Custom scorer can be injected."""- from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig-- custom_scorer = BM25Scorer(k1=2.0)- crusher = SmartCrusher(- config=SmartCrusherConfig(),- scorer=custom_scorer,- )-- assert crusher._scorer is custom_scorer-def test_context_extraction(self):"""Context is extracted from messages."""from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfigdiff --git a/tests/test_relevance_extra.py b/tests/test_relevance_extra.pyindex 62a8fa895..90b6d34e0 100644--- a/tests/test_relevance_extra.py+++ b/tests/test_relevance_extra.py@@ -90,7 +90,7 @@ def test_embedding_numpy_and_model_error_paths(monkeypatch) -> None:assert embedding._cosine_similarity([1, 0], [0, 1]) == 0.0monkeypatch.setattr(EmbeddingScorer, "is_available", classmethod(lambda cls: False))- with pytest.raises(RuntimeError, match="requires sentence-transformers"):+ with pytest.raises(RuntimeError, match="requires fastembed"):EmbeddingScorer()._get_model()diff --git a/tests/test_toin_field_learning.py b/tests/test_toin_field_learning.pydeleted file mode 100644index ae875cd43..000000000--- a/tests/test_toin_field_learning.py+++ /dev/null@@ -1,737 +0,0 @@-"""Tests for TOIN Field-Level Learning.--These tests PROVE that field-level learning actually works:-1. FieldSemantics correctly infers types from retrieval patterns-2. TOIN populates field_semantics from retrievals-3. SmartCrusher uses learned semantics to detect important items-4. End-to-end: important items are preserved based on learned behavior--No hardcoded patterns - all learning is behavior-based.-"""--import hashlib--import pytest--from headroom.telemetry import (- ToolIntelligenceNetwork,- ToolPattern,- ToolSignature,- reset_toin,-)-from headroom.telemetry.models import FieldSemantics--[email protected](autouse=True)-def reset_globals():- """Reset global state before each test."""- reset_toin()- yield- reset_toin()---def _hash_value(value) -> str:- """Hash a value the same way TOIN does."""- value_str = str(value)- return hashlib.sha256(value_str.encode()).hexdigest()[:8]---def _hash_field(field_name: str) -> str:- """Hash a field name the same way TOIN does."""- return hashlib.sha256(field_name.encode()).hexdigest()[:8]---class TestFieldSemanticsLearning:- """Test that FieldSemantics correctly learns from retrieval patterns."""-- def test_identifier_type_inference(self):- """PROVES: Field used with exact-match queries + high uniqueness = identifier."""- fs = FieldSemantics(field_hash="test123")-- # Simulate: user retrieves items by unique IDs (exact match queries)- # Each ID is different - high uniqueness- for i in range(10):- fs.record_retrieval_value(_hash_value(f"id_{i}"), operator="=")-- # Simulate compression stats: all values are unique- fs.record_compression_stats(- unique_values=100,- total_values=100, # uniqueness ratio = 1.0- most_common_value_hash=_hash_value("id_0"),- most_common_frequency=0.01, # No dominant value- )- fs.record_compression_stats(- unique_values=100,- total_values=100,- most_common_value_hash=_hash_value("id_1"),- most_common_frequency=0.01,- )-- # Now infer the type- fs.infer_type()-- # VERIFY: Should be classified as identifier- assert fs.inferred_type == "identifier", (- f"Expected 'identifier' but got '{fs.inferred_type}'. "- "High uniqueness + exact match queries should = identifier"- )- assert fs.confidence > 0.5, "Should have reasonable confidence"-- def test_error_indicator_type_inference(self):- """PROVES: Field with dominant default + retrievals for non-default = error_indicator."""- fs = FieldSemantics(field_hash="status_field")-- # Simulate: most items have status="success" (the default)- # But user only retrieves items with status="error" or "failed"- error_hash = _hash_value("error")- failed_hash = _hash_value("failed")- success_hash = _hash_value("success")-- # User retrieves "error" and "failed" values (non-default)- for _ in range(5):- fs.record_retrieval_value(error_hash, operator="=")- fs.record_retrieval_value(failed_hash, operator="=")-- # Compression stats: 90% have "success" (the default)- fs.record_compression_stats(- unique_values=3, # "success", "error", "failed"- total_values=100,- most_common_value_hash=success_hash,- most_common_frequency=0.9, # 90% are "success"- )- fs.record_compression_stats(- unique_values=3,- total_values=100,- most_common_value_hash=success_hash,- most_common_frequency=0.9,- )-- fs.infer_type()-- # VERIFY: Should be error_indicator- assert fs.inferred_type == "error_indicator", (- f"Expected 'error_indicator' but got '{fs.inferred_type}'. "- "Dominant default + retrieval of non-default = error indicator"- )- assert fs.default_value_hash == success_hash, "Default should be 'success'"- assert fs.confidence > 0.5, "Should have reasonable confidence"-- def test_status_type_inference(self):- """PROVES: Low cardinality + specific values retrieved = status."""- fs = FieldSemantics(field_hash="state_field")-- # Simulate: field has few unique values (low cardinality)- # User retrieves the same few values repeatedly- pending_hash = _hash_value("pending")- active_hash = _hash_value("active")-- for _ in range(6):- fs.record_retrieval_value(pending_hash, operator="=")- for _ in range(4):- fs.record_retrieval_value(active_hash, operator="=")-- # Compression stats: only 5 unique values across 100 items- fs.record_compression_stats(- unique_values=5,- total_values=100, # uniqueness ratio = 0.05 (very low)- most_common_value_hash=None,- most_common_frequency=0.3, # No overwhelming default- )- fs.record_compression_stats(- unique_values=5,- total_values=100,- most_common_value_hash=None,- most_common_frequency=0.3,- )-- fs.infer_type()-- # VERIFY: Should be status- assert fs.inferred_type == "status", (- f"Expected 'status' but got '{fs.inferred_type}'. "- "Low cardinality + specific values retrieved = status"- )-- def test_score_type_inference(self):- """PROVES: Range queries = score type."""- fs = FieldSemantics(field_hash="relevance_field")-- # Simulate: user queries with range operators (top-N behavior)- for _ in range(8):- fs.record_retrieval_value(_hash_value("0.95"), operator=">")- for _ in range(4):- fs.record_retrieval_value(_hash_value("0.90"), operator=">=")-- # Compression stats- fs.record_compression_stats(- unique_values=50,- total_values=100,- most_common_value_hash=None,- most_common_frequency=0.1,- )- fs.record_compression_stats(- unique_values=50,- total_values=100,- most_common_value_hash=None,- most_common_frequency=0.1,- )-- fs.infer_type()-- # VERIFY: Should be score- assert fs.inferred_type == "score", (- f"Expected 'score' but got '{fs.inferred_type}'. "- "Range queries (>, >=) should = score type"- )-- def test_content_type_inference(self):- """PROVES: Contains/text search queries = content type."""- fs = FieldSemantics(field_hash="description_field")-- # Simulate: user does text search on this field- for i in range(10):- fs.record_retrieval_value(_hash_value(f"search_term_{i}"), operator="contains")-- # Compression stats: high uniqueness (different descriptions)- fs.record_compression_stats(- unique_values=90,- total_values=100,- most_common_value_hash=None,- most_common_frequency=0.05,- )- fs.record_compression_stats(- unique_values=90,- total_values=100,- most_common_value_hash=None,- most_common_frequency=0.05,- )-- fs.infer_type()-- # VERIFY: Should be content- assert fs.inferred_type == "content", (- f"Expected 'content' but got '{fs.inferred_type}'. "- "Contains queries should = content type"- )-- def test_is_value_important_for_error_indicator(self):- """PROVES: For error_indicator, non-default values are important."""- fs = FieldSemantics(field_hash="status")-- error_hash = _hash_value("error")- success_hash = _hash_value("success")-- # Set up as error_indicator- fs.inferred_type = "error_indicator"- fs.confidence = 0.8- fs.default_value_hash = success_hash- fs.important_value_hashes = [error_hash]-- # VERIFY- assert fs.is_value_important(error_hash), "Error value should be important"- assert not fs.is_value_important(success_hash), "Default value should NOT be important"-- def test_is_value_important_for_status(self):- """PROVES: For status fields, retrieved values are important."""- fs = FieldSemantics(field_hash="state")-- pending_hash = _hash_value("pending")- unknown_hash = _hash_value("never_retrieved")-- # Set up as status- fs.inferred_type = "status"- fs.confidence = 0.7- fs.value_retrieval_frequency = {pending_hash: 5}-- # VERIFY- assert fs.is_value_important(pending_hash), "Retrieved value should be important"- assert not fs.is_value_important(unknown_hash), (- "Never-retrieved value should NOT be important"- )---class TestTOINFieldLearningIntegration:- """Test that TOIN correctly integrates field-level learning."""-- def test_record_retrieval_populates_field_semantics(self):- """PROVES: record_retrieval with items actually populates field_semantics."""- toin = ToolIntelligenceNetwork()-- # Create a tool signature- items = [- {"id": "123", "status": "ok", "value": 100},- {"id": "456", "status": "error", "value": 200},- ]- sig = ToolSignature.from_items(items)-- # Record retrieval with items - THIS IS WHERE LEARNING HAPPENS- toin.record_retrieval(- tool_signature_hash=sig.structure_hash,- retrieval_type="full",- query="status=error",- query_fields=["status"],- retrieved_items=items,- )-- # VERIFY: pattern should have field_semantics populated- pattern = toin._patterns.get(sig.structure_hash)- assert pattern is not None, "Pattern should exist"- assert len(pattern.field_semantics) > 0, (- f"field_semantics should be populated after retrieval. Got: {pattern.field_semantics}"- )-- # Check that field hashes match expected fields- expected_field_hashes = {_hash_field(f) for f in ["id", "status", "value"]}- actual_field_hashes = set(pattern.field_semantics.keys())- assert expected_field_hashes == actual_field_hashes, (- f"Expected field hashes {expected_field_hashes}, got {actual_field_hashes}"- )-- def test_repeated_retrievals_trigger_type_inference(self):- """PROVES: After multiple retrievals, TOIN infers field types."""- toin = ToolIntelligenceNetwork()-- # Simulate items with status field- items = [- {"status": "success"},- {"status": "success"},- {"status": "success"},- {"status": "error"},- ]- sig = ToolSignature.from_items(items)-- # Simulate 6 retrievals (enough to trigger inference at retrieval 5)- for _ in range(6):- # User always retrieves items with error status- toin.record_retrieval(- tool_signature_hash=sig.structure_hash,- retrieval_type="full",- query="status=error",- query_fields=["status"],- retrieved_items=[{"status": "error"}],- )- # Also record compression to get enough data for inference- toin.record_compression(- tool_signature=sig,- original_count=100,- compressed_count=10,- original_tokens=1000,- compressed_tokens=100,- strategy="top_n",- items=items,- )-- # VERIFY: After enough retrievals, type should be inferred- pattern = toin._patterns.get(sig.structure_hash)- status_hash = _hash_field("status")- assert status_hash in pattern.field_semantics, "Status field should be tracked"-- status_sem = pattern.field_semantics[status_hash]- # The type should be inferred (not unknown) after enough data- assert status_sem.retrieval_count >= 6, (- f"Should have 6+ retrievals, got {status_sem.retrieval_count}"- )-- def test_get_recommendation_includes_field_semantics(self):- """PROVES: get_recommendation returns learned field_semantics."""- toin = ToolIntelligenceNetwork()-- # Set up pattern with learned field_semantics- items = [{"id": "123", "status": "ok"}]- sig = ToolSignature.from_items(items)-- # Record enough data- for i in range(10):- toin.record_retrieval(- tool_signature_hash=sig.structure_hash,- retrieval_type="full",- query=f"id={i}",- query_fields=["id"],- retrieved_items=[{"id": str(i), "status": "ok"}],- )- toin.record_compression(- tool_signature=sig,- original_count=100,- compressed_count=10,- original_tokens=1000,- compressed_tokens=100,- strategy="top_n",- items=items,- )-- # Get recommendation- hint = toin.get_recommendation(sig)-- # VERIFY: hint should include field_semantics- assert hint is not None, "Should get a recommendation"- # field_semantics might be empty if confidence is too low,- # but the attribute should exist- assert hasattr(hint, "field_semantics"), "Hint should have field_semantics attribute"-- def test_field_semantics_persisted_correctly(self):- """PROVES: field_semantics survives to_dict/from_dict round-trip."""- # ToolPattern is already imported at module level from headroom.telemetry-- # Create pattern with field_semantics- pattern = ToolPattern(tool_signature_hash="test123")-- # Add field semantics- fs = FieldSemantics(field_hash="field123")- fs.inferred_type = "error_indicator"- fs.confidence = 0.8- fs.important_value_hashes = ["value1", "value2"]- fs.default_value_hash = "default"- pattern.field_semantics["field123"] = fs-- # Round-trip through dict- d = pattern.to_dict()- pattern2 = ToolPattern.from_dict(d)-- # VERIFY: field_semantics preserved- assert "field123" in pattern2.field_semantics, "field_semantics should be preserved"- fs2 = pattern2.field_semantics["field123"]- assert fs2.inferred_type == "error_indicator"- assert fs2.confidence == 0.8- assert fs2.important_value_hashes == ["value1", "value2"]---class TestSmartCrusherUsesLearnedSemantics:- """Test that SmartCrusher actually uses learned field semantics."""-- def test_detect_items_by_learned_semantics_finds_important_items(self):- """PROVES: _detect_items_by_learned_semantics correctly identifies items."""- from headroom.transforms.smart_crusher import _detect_items_by_learned_semantics-- # Create field semantics that knows "error" is important- status_hash = _hash_field("status")- error_hash = _hash_value("error")- success_hash = _hash_value("success")-- fs = FieldSemantics(field_hash=status_hash)- fs.inferred_type = "error_indicator"- fs.confidence = 0.8- fs.default_value_hash = success_hash- fs.important_value_hashes = [error_hash]- fs.value_retrieval_frequency = {error_hash: 10}-- field_semantics = {status_hash: fs}-- # Test items - index 1 has error status- items = [- {"status": "success", "message": "all good"},- {"status": "error", "message": "something failed"}, # <-- This should be detected- {"status": "success", "message": "also good"},- ]-- # VERIFY- important_indices = _detect_items_by_learned_semantics(items, field_semantics)- assert 1 in important_indices, (- f"Index 1 (error status) should be detected as important. "- f"Got indices: {important_indices}"- )- assert 0 not in important_indices, "Index 0 (success) should not be important"- assert 2 not in important_indices, "Index 2 (success) should not be important"-- def test_detect_items_handles_empty_semantics(self):- """PROVES: Function handles edge cases gracefully."""- from headroom.transforms.smart_crusher import _detect_items_by_learned_semantics-- items = [{"status": "ok"}]-- # Empty semantics- assert _detect_items_by_learned_semantics(items, {}) == []- assert _detect_items_by_learned_semantics(items, None) == []- assert _detect_items_by_learned_semantics([], {"x": FieldSemantics(field_hash="x")}) == []-- def test_detect_items_requires_confidence(self):- """PROVES: Low confidence semantics are ignored."""- from headroom.transforms.smart_crusher import _detect_items_by_learned_semantics-- status_hash = _hash_field("status")- error_hash = _hash_value("error")-- fs = FieldSemantics(field_hash=status_hash)- fs.inferred_type = "error_indicator"- fs.confidence = 0.1 # TOO LOW- fs.important_value_hashes = [error_hash]-- items = [{"status": "error"}]-- # VERIFY: Low confidence = ignored- result = _detect_items_by_learned_semantics(items, {status_hash: fs})- assert result == [], "Low confidence semantics should be ignored"---class TestEndToEndFieldLearning:- """End-to-end tests proving the full learning pipeline works."""-- def test_full_pipeline_learns_and_applies(self):- """PROVES: End-to-end learning from retrieval to compression."""- toin = ToolIntelligenceNetwork()-- # PHASE 1: Learning - User retrieves items with error status- # Simulating: "show me all failed items"- items_with_errors = [- {"id": "1", "status": "success", "data": "..."},- {"id": "2", "status": "error", "data": "..."}, # Retrieved- {"id": "3", "status": "success", "data": "..."},- {"id": "4", "status": "failed", "data": "..."}, # Retrieved- ]- sig = ToolSignature.from_items(items_with_errors)-- # User keeps retrieving error/failed items (learning behavior)- for _ in range(5):- toin.record_retrieval(- tool_signature_hash=sig.structure_hash,- retrieval_type="full",- query="status!=success",- query_fields=["status"],- retrieved_items=[- {"id": "2", "status": "error", "data": "..."},- {"id": "4", "status": "failed", "data": "..."},- ],- )- toin.record_compression(- tool_signature=sig,- original_count=100,- compressed_count=10,- original_tokens=1000,- compressed_tokens=100,- strategy="top_n",- items=[- {"id": str(i), "status": "success" if i % 5 != 0 else "error", "data": "..."}- for i in range(100)- ],- )-- # PHASE 2: Verify learning occurred- pattern = toin._patterns.get(sig.structure_hash)- assert pattern is not None- assert len(pattern.field_semantics) > 0, "Should have learned field semantics"-- # Check status field was learned- status_hash = _hash_field("status")- if status_hash in pattern.field_semantics:- status_sem = pattern.field_semantics[status_hash]- # Error value should be tracked- error_hash = _hash_value("error")- failed_hash = _hash_value("failed")- assert (- error_hash in status_sem.important_value_hashes- or failed_hash in status_sem.important_value_hashes- ), "Error/failed values should be marked as important"-- def test_recommendation_hint_includes_learned_semantics(self):- """PROVES: TOIN recommendation includes learned field semantics for SmartCrusher."""- toin = ToolIntelligenceNetwork()-- # Set up sufficient learning- items = [{"status": "ok", "id": "123"}]- sig = ToolSignature.from_items(items)-- # Create pattern with confident field semantics by directly manipulating internal state- # (This is a test - in real code, patterns are created via record_* methods)- pattern = ToolPattern(tool_signature_hash=sig.structure_hash)- toin._patterns[sig.structure_hash] = pattern-- status_hash = _hash_field("status")- fs = FieldSemantics(field_hash=status_hash)- fs.inferred_type = "error_indicator"- fs.confidence = 0.8 # High confidence- fs.retrieval_count = 10- fs.important_value_hashes = [_hash_value("error")]- pattern.field_semantics[status_hash] = fs-- # Ensure pattern has enough data for recommendation- pattern.total_compressions = 10- pattern.sample_size = 10 # Required by min_samples_for_recommendation- pattern.confidence = 0.5-- # Get recommendation- hint = toin.get_recommendation(sig)-- # VERIFY- assert hint is not None- assert len(hint.field_semantics) > 0, (- f"Recommendation should include field_semantics. Got: {hint.field_semantics}"- )- assert status_hash in hint.field_semantics---class TestFieldSemanticsMemoryBounds:- """Test that memory bounds are enforced."""-- def test_important_values_bounded(self):- """PROVES: important_value_hashes stays within bounds."""- fs = FieldSemantics(field_hash="test")-- # Add more values than MAX_IMPORTANT_VALUES- for i in range(fs.MAX_IMPORTANT_VALUES + 20):- fs.record_retrieval_value(_hash_value(f"value_{i}"))-- # VERIFY: bounded- assert len(fs.important_value_hashes) <= fs.MAX_IMPORTANT_VALUES-- def test_value_frequency_bounded(self):- """PROVES: value_retrieval_frequency stays within bounds."""- fs = FieldSemantics(field_hash="test")-- # Add more values than MAX_VALUE_FREQUENCY_ENTRIES- for i in range(fs.MAX_VALUE_FREQUENCY_ENTRIES + 20):- fs.record_retrieval_value(_hash_value(f"value_{i}"))-- # VERIFY: bounded- assert len(fs.value_retrieval_frequency) <= fs.MAX_VALUE_FREQUENCY_ENTRIES---class TestProductionCodePath:- """Integration tests for the ACTUAL production code path.-- These tests verify that CompressionStore -> TOIN integration works,- not just TOIN in isolation. This is critical because the unit tests- can pass while the production integration is broken.- """-- def test_compression_store_passes_items_to_toin(self):- """PROVES: CompressionStore.process_pending_feedback passes retrieved_items to TOIN.-- This is the integration test that would have caught the original bug- where retrieved_items was never passed to TOIN in production.- """- import json-- from headroom.cache.compression_store import CompressionStore- from headroom.telemetry.toin import get_toin, reset_toin-- reset_toin()- toin = get_toin()-- # Create a store with feedback enabled- store = CompressionStore(max_entries=100, default_ttl=300, enable_feedback=True)-- # Store some compressed content with items that have distinct field values- items = [- {"id": "123", "status": "success", "value": 100},- {"id": "456", "status": "error", "value": 200},- {"id": "789", "status": "success", "value": 300},- ]- compressed_json = json.dumps(items)- original_json = json.dumps(items * 10) # Original was bigger-- # Create a tool signature hash for this structure- sig = ToolSignature.from_items(items)-- hash_key = store.store(- original=original_json,- compressed=compressed_json,- original_tokens=1000,- compressed_tokens=100,- original_item_count=30,- compressed_item_count=3,- tool_name="test_api",- tool_call_id="call_123",- tool_signature_hash=sig.structure_hash,- compression_strategy="top_n",- )-- # Simulate a retrieval (this triggers the feedback loop)- store.retrieve(hash_key, query="status=error")-- # Process pending feedback - THIS IS WHERE THE BUG WAS- store.process_pending_feedback()-- # VERIFY: TOIN should have received the items and learned from them- pattern = toin._patterns.get(sig.structure_hash)- assert pattern is not None, "TOIN should have a pattern for this tool"-- # The key assertion: field_semantics should be populated- # This would have FAILED before the fix because retrieved_items wasn't passed- assert len(pattern.field_semantics) > 0, (- "TOIN should have learned field semantics from the retrieved items. "- "If this fails, CompressionStore is not passing retrieved_items to TOIN."- )-- # Verify specific fields were learned- id_hash = _hash_field("id")- status_hash = _hash_field("status")- value_hash = _hash_field("value")-- learned_fields = set(pattern.field_semantics.keys())- expected_fields = {id_hash, status_hash, value_hash}- assert expected_fields == learned_fields, (- f"Expected fields {expected_fields}, got {learned_fields}"- )-- def test_compression_store_handles_wrapped_arrays(self):- """PROVES: CompressionStore correctly extracts items from wrapped arrays."""- import json-- from headroom.cache.compression_store import CompressionStore- from headroom.telemetry.toin import get_toin, reset_toin-- reset_toin()- toin = get_toin()-- store = CompressionStore(max_entries=100, default_ttl=300, enable_feedback=True)-- # Content wrapped in {"results": [...]} pattern- items = [{"name": "test", "score": 0.95}]- wrapped_content = json.dumps({"results": items, "total": 1})-- sig = ToolSignature.from_items(items)-- hash_key = store.store(- original=wrapped_content,- compressed=wrapped_content,- original_tokens=100,- compressed_tokens=100,- original_item_count=1,- compressed_item_count=1,- tool_name="search_api",- tool_call_id="call_456",- tool_signature_hash=sig.structure_hash,- compression_strategy="top_n",- )-- store.retrieve(hash_key)- store.process_pending_feedback()-- # VERIFY: Items were extracted from wrapped structure- pattern = toin._patterns.get(sig.structure_hash)- assert pattern is not None- assert len(pattern.field_semantics) > 0, (- "TOIN should extract items from wrapped arrays like {'results': [...]}"- )-- def test_compression_store_handles_invalid_json(self):- """PROVES: CompressionStore gracefully handles invalid JSON."""- from headroom.cache.compression_store import CompressionStore- from headroom.telemetry.toin import get_toin, reset_toin-- reset_toin()- get_toin() # Initialize TOIN for feedback loop-- store = CompressionStore(max_entries=100, default_ttl=300, enable_feedback=True)-- # Store invalid JSON content- invalid_json = "not valid json {"-- hash_key = store.store(- original=invalid_json,- compressed=invalid_json,- original_tokens=10,- compressed_tokens=10,- original_item_count=0,- compressed_item_count=0,- tool_name="broken_api",- tool_call_id="call_789",- tool_signature_hash="invalid123",- compression_strategy="none",- )-- # This should not crash- store.retrieve(hash_key)- store.process_pending_feedback()-- # VERIFY: No crash, pattern may or may not exist but no exception- # The main assertion is that we got here without exceptiondiff --git a/tests/test_toin_integration.py b/tests/test_toin_integration.pyindex 4770aa87f..860228781 100644--- a/tests/test_toin_integration.py+++ b/tests/test_toin_integration.py@@ -8,7 +8,6 @@ Tests the complete flow:5. Future compressions get improved recommendations"""-import jsonimport tempfilefrom pathlib import Path@@ -18,18 +17,11 @@ from headroom.cache.compression_store import (get_compression_store,reset_compression_store,)-from headroom.config import CCRConfig-from headroom.telemetry import ToolSignaturefrom headroom.telemetry.toin import (TOINConfig,- ToolIntelligenceNetwork,get_toin,reset_toin,)-from headroom.transforms.smart_crusher import (- SmartCrusher,- SmartCrusherConfig,-)@pytest.fixture@@ -55,332 +47,3 @@ def fresh_store():store = get_compression_store(max_entries=100, default_ttl=300)yield storereset_compression_store()---class TestTOINIntegration:- """Test the full TOIN feedback loop."""-- def test_compression_records_correct_hash(self, fresh_toin, fresh_store):- """Test that SmartCrusher records the correct tool_signature_hash in store."""- # Create test data- items = [{"id": i, "score": 100 - i, "name": f"Item {i}"} for i in range(50)]- content = json.dumps(items)-- # Create a SmartCrusher with CCR enabled- ccr_config = CCRConfig(enabled=True, inject_retrieval_marker=False)- crusher = SmartCrusher(- SmartCrusherConfig(max_items_after_crush=10),- ccr_config=ccr_config,- )-- # Compress the content- crushed, was_modified, info = crusher._smart_crush_content(content, tool_name="test_tool")-- assert was_modified, "Content should be modified by compression"-- # Verify the store has an entry with the correct tool_signature_hash- stats = fresh_store.get_stats()- assert stats["entry_count"] >= 1, "Store should have at least one entry"-- # Get the entry and verify it has tool_signature_hash- # We need to find the hash key from the store- entries = [entry for _, entry in fresh_store._backend.items()]- assert len(entries) >= 1, "Should have at least one entry"-- entry = entries[0]- assert entry.tool_signature_hash is not None, "Entry should have tool_signature_hash"- assert entry.compression_strategy is not None, "Entry should have compression_strategy"-- # Verify the hash matches what ToolSignature would generate- expected_signature = ToolSignature.from_items(items)- assert entry.tool_signature_hash == expected_signature.structure_hash, (- "Stored hash should match ToolSignature.structure_hash"- )-- def test_retrieval_updates_toin_strategy_success(self, fresh_toin, fresh_store):- """Test that retrieval events update TOIN strategy success rates."""- # Create test data- items = [{"id": i, "score": 100 - i, "name": f"Item {i}"} for i in range(50)]- signature = ToolSignature.from_items(items)-- # Record some compressions with fresh_toin- for _ in range(5):- fresh_toin.record_compression(- tool_signature=signature,- original_count=50,- compressed_count=10,- original_tokens=5000,- compressed_tokens=1000,- strategy="smart_sample",- )-- # Get initial strategy success rate- pattern = fresh_toin._patterns.get(signature.structure_hash)- assert pattern is not None, "Pattern should exist after compressions"-- initial_rate = pattern.strategy_success_rates.get("smart_sample", 1.0)- assert initial_rate > 0, "Initial success rate should be positive"-- # Simulate retrieval events (which indicate compression was too aggressive)- for _ in range(3):- fresh_toin.record_retrieval(- tool_signature_hash=signature.structure_hash,- retrieval_type="full",- query="test query",- query_fields=["id"],- strategy="smart_sample",- )-- # Verify success rate decreased- final_rate = pattern.strategy_success_rates.get("smart_sample", 1.0)- assert final_rate < initial_rate, (- f"Success rate should decrease after retrievals: {initial_rate} -> {final_rate}"- )-- def test_full_feedback_loop(self, fresh_toin, fresh_store):- """Test the complete feedback loop: compress → retrieve → learn → recommend."""- # Create test data- items = [{"id": i, "score": 100 - i, "name": f"Item {i}"} for i in range(50)]- content = json.dumps(items)- signature = ToolSignature.from_items(items)-- # Step 1: Compress with SmartCrusher (records to TOIN)- ccr_config = CCRConfig(enabled=True, inject_retrieval_marker=False)- crusher = SmartCrusher(- SmartCrusherConfig(max_items_after_crush=10, use_feedback_hints=True),- ccr_config=ccr_config,- )-- # Multiple compressions to build pattern- for _ in range(5):- crusher._smart_crush_content(content, tool_name="test_tool")-- # Step 2: Verify TOIN has a pattern- pattern = fresh_toin._patterns.get(signature.structure_hash)- assert pattern is not None, "TOIN should have a pattern after compressions"- assert pattern.total_compressions >= 5, "Should have recorded 5 compressions"-- # Step 3: Simulate retrievals (indicating compression was too aggressive)- # Find the stored entry hash- entries = [entry for _, entry in fresh_store._backend.items()]- assert len(entries) > 0, "Should have cached entries"-- # Retrieve multiple times to trigger learning- for entry in entries[:3]:- fresh_store.retrieve(entry.hash, query="find all items")-- # Step 4: Verify TOIN learned from retrievals- pattern = fresh_toin._patterns.get(signature.structure_hash)- assert pattern is not None, "Pattern should exist after retrievals"-- # CRITICAL: Verify field-level learning actually happened- # This assertion would have caught the bug where compression_store- # wasn't passing retrieved_items to TOIN- assert len(pattern.field_semantics) > 0, (- "TOIN should learn field semantics from retrieved items. "- "If this fails, the production code path (CompressionStore -> TOIN) is broken."- )-- # Verify retrieval stats were updated- assert pattern.total_retrievals >= 1, "Should have recorded retrievals"-- recommendation = fresh_toin.get_recommendation(signature, "find all items")-- # After many retrievals, TOIN should recommend more items- # Default is 15-20, but with high retrieval rate it should go higher- assert recommendation.confidence > 0, "Recommendation should have confidence"-- def test_preserve_fields_used_in_compression(self, fresh_toin, fresh_store):- """Test that TOIN preserve_fields are used during compression planning."""- # Create test data with specific fields- items = [{"id": i, "score": 100 - i, "category": f"cat_{i % 3}"} for i in range(50)]- signature = ToolSignature.from_items(items)-- # Record compressions and retrievals that query the "category" field- for _ in range(10):- fresh_toin.record_compression(- tool_signature=signature,- original_count=50,- compressed_count=10,- original_tokens=5000,- compressed_tokens=1000,- strategy="smart_sample",- )-- # Record retrievals that query "category"- for _ in range(5):- fresh_toin.record_retrieval(- tool_signature_hash=signature.structure_hash,- retrieval_type="search",- query="category:cat_1",- query_fields=["category"],- strategy="smart_sample",- )-- # Get recommendation - should now preserve "category" field- recommendation = fresh_toin.get_recommendation(signature, "find all items in cat_1")-- # Verify the recommendation reflects learning- assert recommendation.source in ("local", "network", "default"), (- f"Should have a valid source: {recommendation.source}"- )-- def test_instance_id_stable_across_restarts(self):- """Test that instance_id is stable across restarts."""- reset_toin()- with tempfile.TemporaryDirectory() as tmpdir:- storage_path = str(Path(tmpdir) / "toin_stable.json")-- # Create first TOIN instance- toin1 = ToolIntelligenceNetwork(TOINConfig(storage_path=storage_path))- instance_id_1 = toin1._instance_id-- # Create second instance with same path (simulating restart)- toin2 = ToolIntelligenceNetwork(TOINConfig(storage_path=storage_path))- instance_id_2 = toin2._instance_id-- # Instance IDs should be the same since derived from path- assert instance_id_1 == instance_id_2, (- f"Instance ID should be stable: {instance_id_1} vs {instance_id_2}"- )-- def test_atomic_save(self):- """Test that save() uses atomic writes."""- reset_toin()- with tempfile.TemporaryDirectory() as tmpdir:- storage_path = str(Path(tmpdir) / "toin_atomic.json")- toin = ToolIntelligenceNetwork(TOINConfig(storage_path=storage_path))-- # Create some data- items = [{"id": i, "name": f"test_{i}"} for i in range(10)]- signature = ToolSignature.from_items(items)-- toin.record_compression(- tool_signature=signature,- original_count=10,- compressed_count=5,- original_tokens=1000,- compressed_tokens=500,- strategy="smart_sample",- )-- # Save- toin.save()-- # Verify file exists and is valid JSON- saved_path = Path(storage_path)- assert saved_path.exists(), "Save file should exist"-- with open(saved_path) as f:- data = json.load(f)-- assert "patterns" in data, "Saved data should have patterns"- assert len(data["patterns"]) > 0, "Should have at least one pattern"-- def test_query_patterns_merged_on_import(self):- """Test that query patterns are merged when importing patterns."""- reset_toin()- with tempfile.TemporaryDirectory() as tmpdir:- storage_path = str(Path(tmpdir) / "toin_merge.json")- toin = ToolIntelligenceNetwork(TOINConfig(storage_path=storage_path))-- # Create local pattern- items = [{"id": i, "name": f"test_{i}"} for i in range(10)]- signature = ToolSignature.from_items(items)-- toin.record_compression(- tool_signature=signature,- original_count=10,- compressed_count=5,- original_tokens=1000,- compressed_tokens=500,- strategy="smart_sample",- )-- # Record local query pattern- toin.record_retrieval(- tool_signature_hash=signature.structure_hash,- retrieval_type="search",- query="local_pattern:value",- query_fields=["local_pattern"],- strategy="smart_sample",- )-- # Create import data with different query pattern- import_data = {- "patterns": {- signature.structure_hash: {- "tool_signature_hash": signature.structure_hash,- "total_compressions": 100,- "total_retrievals": 20,- "avg_original_count": 50,- "avg_compressed_count": 10,- "avg_compression_ratio": 0.2,- "retrieval_rate": 0.2,- "common_query_patterns": ["imported_pattern:x"],- "strategy_success_rates": {"smart_sample": 0.8},- "preserve_fields": ["imported_field"],- "user_count": 50,- "last_updated": 1234567890.0,- }- }- }-- # Import patterns- toin.import_patterns(import_data)-- # Verify patterns were merged- pattern = toin._patterns.get(signature.structure_hash)- assert pattern is not None, "Pattern should exist after merge"-- # Check that both query patterns exist- assert "imported_pattern:x" in pattern.common_query_patterns, (- "Imported query pattern should be present"- )---class TestStoreToTOINHash:- """Test the hash correlation between compression_store and TOIN."""-- def test_hash_matches_between_store_and_toin(self, fresh_toin, fresh_store):- """Test that the hash stored in compression_store matches TOIN events."""- # Use 50 items with score field to ensure compression threshold is met- items = [{"id": i, "score": 100 - i, "name": f"Item {i}"} for i in range(50)]- content = json.dumps(items)- signature = ToolSignature.from_items(items)-- # Compress with aggressive settings to ensure items are actually reduced- ccr_config = CCRConfig(enabled=True, inject_retrieval_marker=False)- crusher = SmartCrusher(- SmartCrusherConfig(max_items_after_crush=10),- ccr_config=ccr_config,- )- crushed, was_modified, info = crusher._smart_crush_content(content, tool_name="hash_test")-- # Verify compression actually happened- assert was_modified, f"Content should be modified by compression: {info}"-- # Get the stored hash- entries = [entry for _, entry in fresh_store._backend.items()]- assert len(entries) >= 1, (- f"Should have stored entry. Modified: {was_modified}, Info: {info}"- )- stored_hash = entries[0].tool_signature_hash-- # Verify it matches ToolSignature- assert stored_hash == signature.structure_hash, (- f"Store hash {stored_hash} should match signature {signature.structure_hash}"- )-- # Verify TOIN can receive events for this hash- fresh_toin.record_compression(- tool_signature=signature,- original_count=50,- compressed_count=10,- original_tokens=5000,- compressed_tokens=1000,- strategy="smart_sample",- )-- pattern = fresh_toin._patterns.get(signature.structure_hash)- assert pattern is not None, "TOIN should have pattern for same hash"diff --git a/tests/test_transforms/test_anchor_selector.py b/tests/test_transforms/test_anchor_selector.pydeleted file mode 100644index aa73c4c70..000000000--- a/tests/test_transforms/test_anchor_selector.py+++ /dev/null@@ -1,1186 +0,0 @@-"""Tests for AnchorSelector - Adaptive position-based anchor allocation.--Comprehensive tests covering:-- Adversarial positions: Important data NOT at expected positions-- Size adaptation: Anchor allocation scaling with array size-- Pattern-aware anchoring: Different strategies for different data patterns-- Query-aware anchoring: Query-based anchor adjustment-- Information density: Unique/informative item selection-- Coverage metrics: Distribution coverage verification-- Edge cases: Boundary conditions and special scenarios--These tests verify the AnchorSelector replaces the static "first 3 + last 2"-preservation with adaptive, pattern-aware anchor selection.-"""--import json--import pytest--from headroom import OpenAIProvider, SmartCrusherConfig, Tokenizer-from headroom.transforms.smart_crusher import (- SmartAnalyzer,- SmartCrusher,-)--# =============================================================================-# Test Fixtures-# =============================================================================--_provider = OpenAIProvider()---def get_tokenizer(model: str = "gpt-4o") -> Tokenizer:- """Get a tokenizer for tests using OpenAI provider."""- token_counter = _provider.get_token_counter(model)- return Tokenizer(token_counter, model)---def tokenizer():- """Provide a tokenizer for tests."""- return get_tokenizer()---def default_config():- """Default SmartCrusherConfig for testing."""- return SmartCrusherConfig(- enabled=True,- min_items_to_analyze=3,- min_tokens_to_crush=0, # Always crush for tests- max_items_after_crush=10,- variance_threshold=2.0,- )---def analyzer(default_config):- """SmartAnalyzer instance for testing."""- return SmartAnalyzer(default_config)---def crusher(default_config):- """SmartCrusher instance for testing."""- return SmartCrusher(default_config)---# =============================================================================-# Test Data Generators-# =============================================================================---def generate_uniform_items(n: int, value: int = 0) -> list[dict]:- """Generate array of identical items."""- return [{"id": "same", "value": value, "status": "ok"} for _ in range(n)]---def generate_numbered_items(n: int, with_value: bool = True) -> list[dict]:- """Generate items with sequential IDs for position tracking."""- items = []- for i in range(n):- item = {"id": i, "name": f"Item {i}"}- if with_value:- item["value"] = i * 10- items.append(item)- return items---def generate_time_series_data(- n: int = 50,- spike_positions: list[int] | None = None,- spike_value: float = 1000.0,-) -> list[dict]:- """Generate time series with optional spikes at specific positions."""- items = []- for i in range(n):- value = 100.0 + (i * 0.5) # Slight upward trend- if spike_positions and i in spike_positions:- value = spike_value- items.append(- {- "timestamp": f"2025-01-{(i % 28) + 1:02d}T{(i % 24):02d}:00:00Z",- "value": value,- "metric": "cpu_usage",- }- )- return items---def generate_log_data(- n: int = 50,- error_positions: list[int] | None = None,-) -> list[dict]:- """Generate log-style data with optional errors at specific positions."""- levels = ["INFO", "DEBUG", "WARN"]- items = []- for i in range(n):- level = levels[i % len(levels)]- if error_positions and i in error_positions:- level = "ERROR"- message = f"Critical failure at step {i}: connection timeout"- else:- message = f"Processing request {i} successfully"- items.append(- {- "level": level,- "message": message,- "timestamp": f"2025-01-06T{12 + (i // 60):02d}:{i % 60:02d}:00Z",- }- )- return items---def generate_search_results(n: int = 50) -> list[dict]:- """Generate search results with scores (higher = better, at front)."""- return [- {- "id": f"doc_{i}",- "title": f"Document {i}",- "score": 1.0 - (i * 0.02), # Scores decrease as index increases- "snippet": f"This is a snippet from document {i}...",- }- for i in range(n)- ]---def generate_categorized_items(- categories: dict[str, int],-) -> list[dict]:- """Generate items with specific category distribution.-- Args:- categories: Dict mapping category name to count, e.g. {"A": 30, "B": 30, "C": 40}- """- items = []- i = 0- for category, count in categories.items():- for _ in range(count):- items.append({"id": i, "category": category, "name": f"Item {i}"})- i += 1- return items---# =============================================================================-# Helper Functions-# =============================================================================---def crush_items(- items: list[dict],- tokenizer: Tokenizer,- max_items: int = 10,- query: str = "",- min_items: int = 3,-) -> list[dict]:- """Helper to crush items and return the result list.-- Args:- items: Array of items to compress.- tokenizer: Tokenizer instance.- max_items: Maximum items after crushing.- query: Optional query context.- min_items: Minimum items to analyze.-- Returns:- List of preserved items after crushing.- """- messages = [- {"role": "system", "content": "You are helpful."},- ]-- if query:- messages.append({"role": "user", "content": query})- messages.append(- {- "role": "assistant",- "content": None,- "tool_calls": [- {- "id": "call_1",- "type": "function",- "function": {"name": "search_items", "arguments": "{}"},- }- ],- }- )-- messages.append({"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)})-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=min_items,- max_items_after_crush=max_items,- )- crusher = SmartCrusher(config)- result = crusher.apply(messages, tokenizer)-- # Parse result- tool_content = result.messages[-1]["content"]- json_part = tool_content.split("\n<headroom:")[0]- return json.loads(json_part)---def get_preserved_positions(result: list[dict], original_size: int) -> dict[str, list[int]]:- """Categorize preserved item positions into front/middle/back.-- Args:- result: List of preserved items (must have 'id' field with original index).- original_size: Size of original array.-- Returns:- Dict with 'front', 'middle', 'back' keys containing lists of preserved indices.- """- front_boundary = int(original_size * 0.1)- back_boundary = int(original_size * 0.9)-- positions = {"front": [], "middle": [], "back": []}-- for item in result:- idx = item.get("id")- if idx is None:- continue-- if isinstance(idx, int):- if idx < front_boundary:- positions["front"].append(idx)- elif idx >= back_boundary:- positions["back"].append(idx)- else:- positions["middle"].append(idx)-- return positions---# =============================================================================-# TestAdversarialPositions-# =============================================================================---class TestAdversarialPositions:- """Test scenarios that break 'first 3 + last 2' assumption.-- These tests verify that important data is preserved regardless of position,- not just because it happens to be at the front or back of the array.- """-- def test_important_data_in_middle(self, tokenizer):- """Critical error item at position 50 of 100-item array should be preserved.-- The error item is in the middle where static anchoring would miss it.- It should be preserved due to error detection, not position.- """- items = [{"id": i, "status": "ok", "value": i * 10} for i in range(100)]- # Place critical error in the middle- items[50] = {- "id": 50,- "status": "error",- "error_code": "CRITICAL",- "message": "Connection failed",- }-- crushed = crush_items(items, tokenizer, max_items=15)-- # Error item MUST be preserved regardless of position- preserved_ids = [item["id"] for item in crushed]- assert 50 in preserved_ids, "Critical error at position 50 should be preserved"-- # Verify it's preserved for the right reason (has error indicator)- error_items = [item for item in crushed if item.get("status") == "error"]- assert len(error_items) >= 1, "At least one error item should be preserved"-- def test_spike_not_at_boundaries(self, tokenizer):- """Numeric spike at position 75 of 100-item array must be preserved.-- The spike represents an anomaly that should be detected statistically,- not missed because it's not at the front or back.- """- items = generate_time_series_data(100, spike_positions=[75], spike_value=1000.0)- # Add id field for tracking- for i, item in enumerate(items):- item["id"] = i-- crushed = crush_items(items, tokenizer, max_items=15)-- # Spike MUST be preserved as anomaly- has_spike = any(item.get("value", 0) > 500 for item in crushed)- assert has_spike, "Anomalous spike at position 75 should be preserved"-- # Verify the specific position was kept- preserved_ids = [item.get("id") for item in crushed]- assert 75 in preserved_ids, "Position 75 with spike should be in result"-- def test_multiple_spikes_scattered(self, tokenizer):- """Multiple spikes at positions 25, 50, 75 should all be preserved."""- spike_positions = [25, 50, 75]- items = generate_time_series_data(100, spike_positions=spike_positions, spike_value=999.0)- for i, item in enumerate(items):- item["id"] = i-- crushed = crush_items(items, tokenizer, max_items=15)-- # All spikes should be preserved- preserved_ids = {item.get("id") for item in crushed}- for pos in spike_positions:- assert pos in preserved_ids, f"Spike at position {pos} should be preserved"-- def test_first_items_identical(self, tokenizer):- """First 10 identical items - should NOT waste all anchor slots on them.-- When first N items are identical, we should keep at most 1-2 of them,- not waste 3 anchor slots on duplicates.- """- # First 10 items are identical- identical_items = [{"id": "same", "value": 0, "type": "duplicate"} for _ in range(10)]- # Rest are unique- unique_items = [{"id": f"unique_{i}", "value": i * 10, "type": "unique"} for i in range(90)]- items = identical_items + unique_items-- crushed = crush_items(items, tokenizer, max_items=10)-- # Should NOT have multiple identical items- same_count = sum(1 for item in crushed if item.get("id") == "same")- assert same_count <= 2, f"Got {same_count} identical items, expected at most 2"-- # Should have some unique items- unique_count = sum(1 for item in crushed if item.get("type") == "unique")- assert unique_count >= 5, f"Expected at least 5 unique items, got {unique_count}"-- def test_last_items_identical(self, tokenizer):- """Last 10 identical items - should NOT waste anchor slots.-- Similar to above but for back anchors - when last N items are identical,- we should preserve variety instead of duplicates.- """- # First 90 items are unique- unique_items = [{"id": f"unique_{i}", "value": i * 10, "type": "unique"} for i in range(90)]- # Last 10 items are identical- identical_items = [{"id": "same", "value": 100, "type": "duplicate"} for _ in range(10)]- items = unique_items + identical_items-- crushed = crush_items(items, tokenizer, max_items=10)-- # Should NOT have multiple identical items- same_count = sum(1 for item in crushed if item.get("id") == "same")- assert same_count <= 2, f"Got {same_count} identical items, expected at most 2"-- def test_relevant_item_in_middle(self, tokenizer):- """Item matching query in middle position must be found.-- When user queries for a specific item that's at position 42,- it should be preserved even though it's not at front/back.- """- items = [{"id": i, "name": f"item_{i}", "status": "active"} for i in range(100)]- # Put target item in the middle- items[42]["name"] = "target_special_item"- items[42]["description"] = "This is what the user is looking for"-- crushed = crush_items(- items,- tokenizer,- max_items=10,- query="Find target_special_item",- )-- # Query-matched item MUST be preserved- has_target = any("target_special_item" in item.get("name", "") for item in crushed)- assert has_target, "Item matching query should be preserved regardless of position"-- def test_error_in_middle_of_otherwise_uniform_data(self, tokenizer):- """Single error in middle of 100 identical OK items must be preserved."""- items = [{"id": i, "status": "ok", "data": "normal"} for i in range(100)]- items[47] = {"id": 47, "status": "failed", "error": "Unexpected failure"}-- crushed = crush_items(items, tokenizer, max_items=15)-- # Error must be preserved- error_items = [item for item in crushed if item.get("status") == "failed"]- assert len(error_items) >= 1, "Error item at position 47 should be preserved"- assert error_items[0]["id"] == 47, "The specific error item should be preserved"---# =============================================================================-# TestSizeAdaptation-# =============================================================================---class TestSizeAdaptation:- """Test that anchor allocation scales with array size.-- Larger arrays should allocate proportionally more anchor slots- to ensure adequate coverage across the data range.- """-- @pytest.mark.parametrize(- "size,max_items,expected_min_anchors",- [- (20, 10, 3), # Small array: at least 3 anchors (front + back)- (100, 15, 4), # Medium array: at least 4 anchors- (500, 20, 5), # Large array: at least 5 anchors- (2000, 25, 6), # Very large: at least 6 anchors- ],- )- def test_anchor_count_scales(self, tokenizer, size, max_items, expected_min_anchors):- """Anchor count should increase with array size.-- Verifies that the number of items from boundary regions (front 10%, back 10%)- increases as the array size grows.- """- items = generate_numbered_items(size)-- crushed = crush_items(items, tokenizer, max_items=max_items)-- # Count items from first 10% and last 10%- front_boundary = int(size * 0.1)- back_boundary = int(size * 0.9)-- anchor_count = sum(- 1 for item in crushed if item["id"] < front_boundary or item["id"] >= back_boundary- )-- assert anchor_count >= expected_min_anchors, (- f"Array of size {size} should have at least {expected_min_anchors} "- f"anchors from boundary regions, got {anchor_count}"- )-- def test_small_array_high_preservation(self, tokenizer):- """Small arrays (< max_items) should preserve most/all items.-- When the array is smaller than max_items, there's no need to drop items.- """- items = generate_numbered_items(8)-- # max_items=20 is larger than array size- crushed = crush_items(items, tokenizer, max_items=20)-- # Should preserve all or nearly all items- assert len(crushed) >= 7, f"Small array should preserve most items, got {len(crushed)}"-- def test_small_array_exact_size(self, tokenizer):- """Array exactly at max_items should preserve all items."""- items = generate_numbered_items(10)-- crushed = crush_items(items, tokenizer, max_items=10)-- # Should preserve all items- assert len(crushed) == 10, "Array at max_items limit should keep all items"-- def test_large_array_efficient_sampling(self, tokenizer):- """Large arrays should sample efficiently across all positions.-- A 500-item array crushed to 20 items should have representation- from front, middle, and back regions.- """- items = generate_numbered_items(500)-- crushed = crush_items(items, tokenizer, max_items=20)-- positions = get_preserved_positions(crushed, 500)-- # Should have representation from all regions- has_front = len(positions["front"]) >= 1- has_back = len(positions["back"]) >= 1- # Middle representation is optional but preferred for large arrays- has_middle = len(positions["middle"]) >= 1-- assert has_front, "Large array should preserve items from front"- assert has_back, "Large array should preserve items from back"- # Middle coverage is important for large arrays- assert has_middle, "Large array should have some middle representation"-- def test_very_large_array_coverage(self, tokenizer):- """Very large array (1000+) should have good position distribution."""- items = generate_numbered_items(1000)-- crushed = crush_items(items, tokenizer, max_items=25)-- positions = get_preserved_positions(crushed, 1000)-- # Calculate coverage spread- all_positions = positions["front"] + positions["middle"] + positions["back"]- if len(all_positions) >= 2:- spread = max(all_positions) - min(all_positions)- # Should span at least 80% of the array- assert spread >= 800, f"Preserved items should span array, spread was {spread}"---# =============================================================================-# TestPatternAwareAnchoring-# =============================================================================---class TestPatternAwareAnchoring:- """Test pattern-specific anchor strategies.-- Different data patterns (search results, logs, time series) should- use different anchor weighting strategies.- """-- def test_search_results_front_heavy(self, tokenizer):- """Search results with scores should preserve more from front (high scores).-- Search results are sorted by relevance score, so top items are most important.- """- items = generate_search_results(100)- for i, item in enumerate(items):- item["idx"] = i # Track original position-- crushed = crush_items(items, tokenizer, max_items=12)-- # Count items by their original position- front_count = sum(1 for item in crushed if item.get("idx", 100) < 30)- back_count = sum(1 for item in crushed if item.get("idx", 0) >= 70)-- # For search results, front (high scores) should dominate- assert front_count > back_count, (- f"Search results should preserve more front items (high scores), "- f"got front={front_count}, back={back_count}"- )-- # Top results should definitely be present- ids = [item["id"] for item in crushed]- assert "doc_0" in ids, "Top search result should be preserved"- assert "doc_1" in ids, "Second search result should be preserved"-- def test_logs_back_heavy(self, tokenizer):- """Logs with timestamps should preserve more recent items (back of array).-- Logs are typically ordered chronologically, with recent items at the end.- """- items = generate_log_data(100)- for i, item in enumerate(items):- item["idx"] = i-- crushed = crush_items(items, tokenizer, max_items=12)-- # For logs, back (recent) should be emphasized- # Extract indices - use 'idx' field we added- indices = [item.get("idx", 0) for item in crushed]-- recent_count = sum(1 for idx in indices if idx >= 70)- old_count = sum(1 for idx in indices if idx < 30)-- # Recent logs should be at least as represented as old logs- assert recent_count >= old_count, (- f"Logs should preserve recent items, got recent={recent_count}, old={old_count}"- )-- def test_time_series_balanced(self, tokenizer):- """Time series should have balanced front/back representation.-- For trend analysis, we need both the start and end of the time series.- """- items = generate_time_series_data(100)- for i, item in enumerate(items):- item["id"] = i-- crushed = crush_items(items, tokenizer, max_items=12)-- positions = get_preserved_positions(crushed, 100)-- front_count = len(positions["front"])- back_count = len(positions["back"])-- # Should be relatively balanced for time series (within 2:1 ratio)- if front_count > 0 and back_count > 0:- ratio = max(front_count, back_count) / min(front_count, back_count)- assert ratio <= 3, f"Time series should have balanced anchors, ratio was {ratio}"-- def test_generic_distributed(self, tokenizer):- """Generic data should sample across all positions.-- When pattern is unknown, sampling should be distributed rather than- heavily weighted to any particular region.- """- items = generate_numbered_items(100)-- crushed = crush_items(items, tokenizer, max_items=15)-- positions = get_preserved_positions(crushed, 100)-- # Should have items from multiple regions- regions_with_items = sum(1 for region in ["front", "middle", "back"] if positions[region])-- assert regions_with_items >= 2, (- f"Generic data should cover multiple regions, got {regions_with_items}"- )---# =============================================================================-# TestQueryAwareAnchoring-# =============================================================================---class TestQueryAwareAnchoring:- """Test query-based anchor adjustment.-- User queries containing temporal keywords should shift anchor weighting.- """-- def test_latest_query_shifts_to_back(self, tokenizer):- """'Latest' in query should preserve more recent items."""- items = [{"id": i, "created": f"2024-01-{i:02d}"} for i in range(1, 31)]-- crushed = crush_items(- items,- tokenizer,- max_items=8,- query="Show me the latest entries",- )-- ids = [item["id"] for item in crushed]- recent_count = sum(1 for id in ids if id > 20)-- # Should have multiple recent items due to "latest" keyword- assert recent_count >= 2, (- f"Query with 'latest' should preserve recent items, got {recent_count}"- )-- def test_recent_query_shifts_to_back(self, tokenizer):- """'Recent' in query should preserve more recent items."""- items = generate_log_data(50)- for i, item in enumerate(items):- item["idx"] = i-- crushed = crush_items(- items,- tokenizer,- max_items=10,- query="Show me recent log entries",- )-- indices = [item.get("idx", 0) for item in crushed]- recent_count = sum(1 for idx in indices if idx >= 35)-- assert recent_count >= 2, "Query with 'recent' should have multiple recent items"-- def test_first_query_shifts_to_front(self, tokenizer):- """'First' in query should preserve earlier items."""- items = [{"id": i, "created": f"2024-01-{i:02d}"} for i in range(1, 31)]-- crushed = crush_items(- items,- tokenizer,- max_items=8,- query="Show me the first entries",- )-- ids = [item["id"] for item in crushed]- early_count = sum(1 for id in ids if id < 10)-- # Should have multiple early items due to "first" keyword- assert early_count >= 2, (- f"Query with 'first' should preserve early items, got {early_count}"- )-- def test_oldest_query_shifts_to_front(self, tokenizer):- """'Oldest' in query should preserve earlier items."""- items = generate_numbered_items(50)-- crushed = crush_items(- items,- tokenizer,- max_items=10,- query="Find the oldest records",- )-- ids = [item["id"] for item in crushed]- early_count = sum(1 for id in ids if id < 15)-- assert early_count >= 3, "Query with 'oldest' should have multiple early items"-- def test_specific_id_query_finds_item(self, tokenizer):- """Query for specific ID should find it regardless of position."""- items = [{"id": f"item_{i:04d}", "value": i} for i in range(200)]-- crushed = crush_items(- items,- tokenizer,- max_items=10,- query="Find item_0123",- )-- # Item at position 123 should be found- ids = [item["id"] for item in crushed]- assert "item_0123" in ids, "Specific ID query should find the item"-- def test_no_query_uses_default_weights(self, tokenizer):- """Without query, use pattern-based defaults."""- items = generate_numbered_items(100)-- crushed = crush_items(items, tokenizer, max_items=15)-- # Without query, should use default anchoring (some front, some back)- positions = get_preserved_positions(crushed, 100)-- assert len(positions["front"]) >= 1, "Default should include front items"- assert len(positions["back"]) >= 1, "Default should include back items"---# =============================================================================-# TestInformationDensity-# =============================================================================---class TestInformationDensity:- """Test information-density based selection.-- Items with unique or rare properties should be preferred over common ones.- """-- def test_unique_items_preferred(self, tokenizer):- """Items with rare field values should be preferred over common ones.-- When most items have status="ok" but a few have status="warning",- the warning items should be preserved as they're more informative.- """- items = []- for i in range(100):- status = "warning" if i in [25, 50, 75] else "ok"- items.append({"id": i, "status": status, "value": i})-- crushed = crush_items(items, tokenizer, max_items=15)-- # Warning items (rare status) should be preserved- warning_count = sum(1 for item in crushed if item.get("status") == "warning")- assert warning_count >= 2, f"Rare status items should be preferred, got {warning_count}"-- def test_dedup_identical_items(self, tokenizer):- """Identical items should be deduplicated in anchor selection.-- If positions 0-5 all have identical content, we shouldn't keep all of them.- """- # First 6 items are identical- items = [{"id": "dup", "value": 0, "constant": "same"} for _ in range(6)]- # Rest are unique- items.extend([{"id": f"uniq_{i}", "value": i * 10, "unique": True} for i in range(94)])-- crushed = crush_items(items, tokenizer, max_items=12)-- # Should not have many identical items- dup_count = sum(1 for item in crushed if item.get("id") == "dup")- unique_count = sum(1 for item in crushed if item.get("unique"))-- assert dup_count <= 2, f"Should deduplicate identical items, got {dup_count}"- assert unique_count >= 6, f"Should prefer unique items, got {unique_count}"-- def test_structural_outliers_preferred(self, tokenizer):- """Items with different structure should be preferred.-- An item with extra fields (like an error with stack trace) should be- preferred over uniform items.- """- items = [{"id": i, "status": "ok"} for i in range(100)]- # Add a structurally different item in the middle- items[42] = {- "id": 42,- "status": "error",- "error_message": "Something went wrong",- "stack_trace": "at line 123...",- "error_code": "ERR_001",- }-- crushed = crush_items(items, tokenizer, max_items=15)-- # Structural outlier should be preserved- outlier = [item for item in crushed if item.get("stack_trace")]- assert len(outlier) >= 1, "Structurally different item should be preserved"-- def test_diverse_values_over_uniform(self, tokenizer):- """When selecting from candidates, prefer diverse values."""- items = []- # Create items with varying diversity- for i in range(100):- items.append(- {- "id": i,- "type": "common" if i < 90 else f"rare_type_{i}",- "value": i,- }- )-- crushed = crush_items(items, tokenizer, max_items=15)-- # Should have some rare types- rare_types = [item for item in crushed if "rare" in item.get("type", "")]- assert len(rare_types) >= 1, "Rare type values should be preserved"---# =============================================================================-# TestCoverageMetrics-# =============================================================================---class TestCoverageMetrics:- """Test that preserved items represent the full distribution.-- Compression should maintain coverage of value ranges, categories, and time.- """-- def test_value_range_coverage(self, tokenizer):- """Preserved items should cover the value range."""- items = [{"id": i, "value": i} for i in range(100)]-- crushed = crush_items(items, tokenizer, max_items=12)-- values = [item["value"] for item in crushed]-- # Should cover most of the range [0, 100]- assert min(values) < 10, "Should have low values"- assert max(values) > 90, "Should have high values"-- # Should have some middle values too- middle_count = sum(1 for v in values if 30 < v < 70)- assert middle_count >= 1, "Should have some middle-range values"-- def test_category_coverage(self, tokenizer):- """Preserved items should represent multiple categories."""- items = generate_categorized_items({"A": 30, "B": 30, "C": 40})-- crushed = crush_items(items, tokenizer, max_items=12)-- categories = {item["category"] for item in crushed}-- # Should have at least 2 of 3 categories represented- assert len(categories) >= 2, f"Should cover multiple categories, got {categories}"-- def test_category_proportional_representation(self, tokenizer):- """Category distribution should roughly reflect original proportions."""- items = generate_categorized_items({"major": 80, "minor": 20})-- crushed = crush_items(items, tokenizer, max_items=15)-- major_count = sum(1 for item in crushed if item["category"] == "major")- minor_count = sum(1 for item in crushed if item["category"] == "minor")-- # Major category should have more items, but minor should be represented- assert major_count > minor_count, "Major category should dominate"- assert minor_count >= 1, "Minor category should still be represented"-- def test_temporal_coverage(self, tokenizer):- """Preserved items should span the time range."""- # Create items spanning 12 months- items = [- {"id": i, "timestamp": f"2024-{(i % 12) + 1:02d}-15", "event": f"event_{i}"}- for i in range(60)- ]-- crushed = crush_items(items, tokenizer, max_items=12)-- months = [int(item["timestamp"][5:7]) for item in crushed]-- # Should span a significant portion of the year- month_range = max(months) - min(months)- assert month_range >= 5, f"Should span multiple months, got range of {month_range}"-- def test_numeric_distribution_coverage(self, tokenizer):- """Preserved numeric values should represent the distribution."""- # Create items with bimodal distribution- items = []- for i in range(50):- items.append({"id": i, "value": 10 + (i % 5)}) # Low cluster: 10-15- for i in range(50):- items.append({"id": 50 + i, "value": 90 + (i % 5)}) # High cluster: 90-95-- crushed = crush_items(items, tokenizer, max_items=12)-- values = [item["value"] for item in crushed]-- # Should have items from both clusters- low_cluster = [v for v in values if v < 20]- high_cluster = [v for v in values if v > 85]-- assert len(low_cluster) >= 1, "Should have items from low cluster"- assert len(high_cluster) >= 1, "Should have items from high cluster"---# =============================================================================-# TestEdgeCases-# =============================================================================---class TestEdgeCases:- """Edge cases and boundary conditions."""-- def test_empty_array(self, tokenizer):- """Empty array should return empty result."""- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": "[]"},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- )- crusher = SmartCrusher(config)- result = crusher.apply(messages, tokenizer)-- tool_content = result.messages[-1]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- assert crushed == [], "Empty array should return empty result"-- def test_single_item(self, tokenizer):- """Single item array should preserve the item."""- items = [{"id": 0, "name": "Only item"}]-- crushed = crush_items(items, tokenizer, max_items=10, min_items=1)-- assert len(crushed) == 1, "Single item should be preserved"- assert crushed[0]["name"] == "Only item"-- def test_two_items(self, tokenizer):- """Two item array should preserve both items."""- items = [{"id": 0, "name": "First"}, {"id": 1, "name": "Second"}]-- crushed = crush_items(items, tokenizer, max_items=10, min_items=1)-- assert len(crushed) == 2, "Both items should be preserved"-- def test_all_identical_items(self, tokenizer):- """Array of all identical items should keep minimal anchors."""- items = [{"id": "same", "value": 42, "status": "ok"} for _ in range(50)]-- crushed = crush_items(items, tokenizer, max_items=10)-- # Should not keep all 10 identical items - that would be wasteful- # But some anchors are needed for structure- assert len(crushed) <= 10, "Should respect max_items"- # All items are identical, so any subset is equivalent-- def test_max_items_greater_than_array(self, tokenizer):- """max_items > array size should return all items."""- items = generate_numbered_items(8)-- crushed = crush_items(items, tokenizer, max_items=20)-- assert len(crushed) == 8, "Should return all items when max_items > array size"-- def test_max_items_equals_array(self, tokenizer):- """max_items == array size should return all items."""- items = generate_numbered_items(15)-- crushed = crush_items(items, tokenizer, max_items=15)-- assert len(crushed) == 15, "Should return all items when max_items == array size"-- def test_all_items_have_errors(self, tokenizer):- """When all items have errors, quality guarantee preserves all.-- Current behavior: Error preservation takes precedence over max_items.- All error items are preserved to avoid losing critical information.- This is a deliberate design choice documented in SmartCrusher.-- NOTE: Future AnchorSelector may implement error deduplication or sampling- for cases where all items are errors, but that requires careful design.- """- items = [{"id": i, "status": "error", "error_code": f"ERR_{i}"} for i in range(50)]-- crushed = crush_items(items, tokenizer, max_items=15)-- # Current behavior: all errors are preserved (quality guarantee)- # This is intentional - we don't want to lose error information- assert len(crushed) == 50, "All error items should be preserved (quality guarantee)"-- def test_none_values_handled(self, tokenizer):- """Items with None values should be handled gracefully."""- items = [- {"id": i, "value": None if i % 3 == 0 else i * 10, "name": f"Item {i}"}- for i in range(30)- ]-- crushed = crush_items(items, tokenizer, max_items=10)-- # Should not crash, should return valid items- assert len(crushed) > 0, "Should handle None values"- assert all(isinstance(item, dict) for item in crushed)-- def test_mixed_types_in_array(self, tokenizer):- """Array with mixed item structures should be handled."""- items = [- {"id": 0, "simple": True},- {"id": 1, "nested": {"deep": {"value": 42}}},- {"id": 2, "list_field": [1, 2, 3]},- {"id": 3, "mixed": {"a": [1, 2], "b": "text"}},- ]- # Add more simple items to trigger crushing- items.extend([{"id": i, "simple": True} for i in range(4, 20)])-- crushed = crush_items(items, tokenizer, max_items=8, min_items=3)-- # Should preserve structurally interesting items- assert len(crushed) > 0, "Should handle mixed structures"-- def test_unicode_content_preserved(self, tokenizer):- """Unicode content should be preserved correctly."""- # Use the full Unicode codepoint for rocket emoji (U+1F680)- rocket_emoji = "\U0001f680" # Full codepoint, not surrogate pair- items = [- {"id": i, "name": f"Item {i} - \u4e2d\u6587 \u65e5\u672c\u8a9e {rocket_emoji}"}- for i in range(20)- ]-- crushed = crush_items(items, tokenizer, max_items=10)-- # Unicode should be preserved- for item in crushed:- assert "\u4e2d\u6587" in item["name"], "Chinese characters should be preserved"- assert rocket_emoji in item["name"], "Emoji should be preserved"-- def test_very_long_string_values(self, tokenizer):- """Items with very long string values should be handled."""- items = [{"id": i, "data": "x" * 10000 if i == 10 else "short"} for i in range(30)]-- crushed = crush_items(items, tokenizer, max_items=10)-- # Should not crash- assert len(crushed) > 0, "Should handle long strings"-- def test_deeply_nested_items(self, tokenizer):- """Deeply nested items should be handled without stack overflow."""-- def create_nested(depth: int) -> dict:- if depth == 0:- return {"value": "leaf"}- return {"nested": create_nested(depth - 1)}-- items = [{"id": i, "deep": create_nested(10)} for i in range(20)]-- crushed = crush_items(items, tokenizer, max_items=10)-- assert len(crushed) > 0, "Should handle deeply nested items"---# =============================================================================-# TestAnchorConfigBehavior-# =============================================================================---class TestAnchorConfigBehavior:- """Test configuration-driven anchor behavior."""-- def test_high_max_items_reduces_compression(self, tokenizer):- """Higher max_items should preserve more items.-- NOTE: Data must have "importance signals" (errors, anomalies) to trigger- compression. Generic unique items without signals are SKIPPED by the- crushability analysis (conservative behavior to avoid losing entities).-- This test uses search results which always compress based on score.- """- # Use search results - they always trigger TOP_N compression- items = generate_search_results(100)-- crushed_low = crush_items(items, tokenizer, max_items=8)- crushed_high = crush_items(items, tokenizer, max_items=25)-- assert len(crushed_high) > len(crushed_low), (- f"Higher max_items should preserve more items, "- f"got low={len(crushed_low)}, high={len(crushed_high)}"- )-- def test_min_items_to_analyze_threshold(self, tokenizer):- """Arrays below min_items_to_analyze should not be crushed."""- items = generate_numbered_items(5)-- # Set min_items_to_analyze higher than array size- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=10, # Higher than array size- max_items_after_crush=3,- )- crusher = SmartCrusher(config)- result = crusher.apply(messages, tokenizer)-- tool_content = result.messages[-1]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- # Should not crush because array is below threshold- assert len(crushed) == 5, "Array below min_items_to_analyze should not be crushed"---# =============================================================================-# TestPreservationGuarantees-# =============================================================================---class TestPreservationGuarantees:- """Test hard guarantees about what must be preserved."""-- def test_errors_always_preserved(self, tokenizer):- """Error items must ALWAYS be preserved, regardless of position or count."""- items = [{"id": i, "status": "ok"} for i in range(100)]- # Scatter errors throughout- error_positions = [10, 25, 45, 65, 85]- for pos in error_positions:- items[pos] = {"id": pos, "status": "error", "error": f"Error at {pos}"}-- crushed = crush_items(items, tokenizer, max_items=15)-- # All errors should be preserved- error_count = sum(1 for item in crushed if item.get("status") == "error")- assert error_count == len(error_positions), (- f"All {len(error_positions)} errors should be preserved, got {error_count}"- )-- def test_anomalies_always_preserved(self, tokenizer):- """Numeric anomalies (statistical outliers) must be preserved."""- items = [{"id": i, "value": 100 + (i % 5)} for i in range(100)]- # Add anomalies- items[33]["value"] = 9999- items[66]["value"] = -9999-- crushed = crush_items(items, tokenizer, max_items=15)-- values = [item["value"] for item in crushed]-- # Anomalies should be present- assert 9999 in values, "High anomaly should be preserved"- assert -9999 in values, "Low anomaly should be preserved"-- def test_query_matches_always_preserved(self, tokenizer):- """Items matching user query must be preserved."""- items = [{"id": i, "name": f"generic_item_{i}"} for i in range(100)]- items[55]["name"] = "special_target_item"- items[55]["important"] = True-- crushed = crush_items(- items,- tokenizer,- max_items=10,- query="Find special_target_item",- )-- # Target should be found- names = [item["name"] for item in crushed]- assert "special_target_item" in names, "Query-matched item must be preserved"-- def test_change_points_preserved(self, tokenizer):- """Items around detected change points should be preserved."""- # Create data with clear step change- items = []- for i in range(60):- value = 100.0 if i < 30 else 300.0 # Step change at index 30- items.append({"id": i, "value": value, "timestamp": f"2025-01-{(i % 28) + 1:02d}"})-- crushed = crush_items(items, tokenizer, max_items=12)-- # Should have items from around the change point- preserved_ids = [item["id"] for item in crushed]-- # Check for items near the change point (indices 28-32)- near_change = sum(1 for id in preserved_ids if 25 <= id <= 35)- assert near_change >= 1, "Items near change point should be preserved"diff --git a/tests/test_transforms/test_smart_crusher.py b/tests/test_transforms/test_smart_crusher.pydeleted file mode 100644index 238690e7d..000000000--- a/tests/test_transforms/test_smart_crusher.py+++ /dev/null@@ -1,1359 +0,0 @@-"""Tests for SmartCrusher transform.--Comprehensive tests covering:-- SmartAnalyzer: Statistical analysis of arrays-- SmartCrusher: Intelligent compression with Safe V1 Recipe-- RelevanceScoring: Context extraction and item matching-- Edge cases: Malformed JSON, nested arrays, different message formats-"""--import json--import pytest--from headroom import (- OpenAIProvider,- RelevanceScorerConfig,- SmartCrusherConfig,- Tokenizer,-)-from headroom.relevance import RelevanceScore, RelevanceScorer-from headroom.transforms.smart_crusher import (- CompressionStrategy,- SmartAnalyzer,- SmartCrusher,-)--# =============================================================================-# Test Fixtures-# =============================================================================--# Create a shared provider for tests-_provider = OpenAIProvider()---def get_tokenizer(model: str = "gpt-4o") -> Tokenizer:- """Get a tokenizer for tests using OpenAI provider."""- token_counter = _provider.get_token_counter(model)- return Tokenizer(token_counter, model)---def tokenizer():- """Provide a tokenizer for tests."""- return get_tokenizer()---def default_config():- """Default SmartCrusherConfig for testing."""- return SmartCrusherConfig(- enabled=True,- min_items_to_analyze=3,- min_tokens_to_crush=0, # Always crush for tests- max_items_after_crush=10,- variance_threshold=2.0,- )---def analyzer(default_config):- """SmartAnalyzer instance for testing."""- return SmartAnalyzer(default_config)---def crusher(default_config):- """SmartCrusher instance for testing."""- return SmartCrusher(default_config)---# =============================================================================-# Test Data Generators-# =============================================================================---def generate_time_series_data(n: int = 20, with_spike: bool = False) -> list[dict]:- """Generate time series data with optional anomaly."""- data = []- for i in range(n):- value = 100.0 + (i * 0.5) # Slight upward trend- if with_spike and i == n // 2:- value = 500.0 # Spike in the middle- data.append(- {- "timestamp": f"2025-01-{(i % 28) + 1:02d}T12:00:00Z",- "value": value,- "metric": "cpu_usage",- }- )- return data---def generate_log_data(n: int = 20, with_errors: bool = False) -> list[dict]:- """Generate log-style data with optional errors."""- data = []- levels = ["INFO", "DEBUG", "WARN"]- for i in range(n):- level = levels[i % len(levels)]- if with_errors and i in [5, 15]:- level = "ERROR"- message = f"Connection failed: timeout after 30s (attempt {i})"- else:- message = f"Processing request {i} successfully"- data.append(- {- "level": level,- "message": message,- "timestamp": f"2025-01-06T{12 + (i // 60):02d}:{i % 60:02d}:00Z",- }- )- return data---def generate_search_results(n: int = 20) -> list[dict]:- """Generate search results with scores."""- return [- {- "id": f"doc_{i}",- "title": f"Document {i}",- "score": 1.0 - (i * 0.05),- "snippet": f"This is a snippet from document {i}...",- }- for i in range(n)- ]---def generate_generic_data(- n: int = 20,- constant_field: bool = False,- with_signals: bool = False,-) -> list[dict]:- """Generate generic array data.-- Args:- n: Number of items to generate- constant_field: If True, type field is constant "product"- with_signals: If True, adds importance signals (errors, anomalies)- to enable crushing with new statistical detection- """- items = []- for i in range(n):- item = {- "id": i,- "name": f"Item {i}",- "type": "product" if constant_field else f"type_{i % 3}",- "active": True if constant_field else (i % 2 == 0),- }- if with_signals:- item["value"] = 100.0- # Add some errors- if i == n // 4:- item["error"] = f"Error at {i}"- # Add some anomalies- if i == n // 2:- item["value"] = 99999.0- items.append(item)- return items---# =============================================================================-# TestSmartAnalyzer-# =============================================================================---class TestSmartAnalyzer:- """Tests for SmartAnalyzer class."""-- def test_analyze_empty_array(self, analyzer):- """Empty array should return analysis with no field stats."""- result = analyzer.analyze_array([])-- assert result.item_count == 0- assert result.field_stats == {}- assert result.detected_pattern == "generic"- assert result.recommended_strategy == CompressionStrategy.NONE- assert result.constant_fields == {}-- def test_analyze_single_item(self, analyzer):- """Single item array should return analysis but no compression."""- items = [{"id": 1, "name": "Test"}]- result = analyzer.analyze_array(items)-- assert result.item_count == 1- assert "id" in result.field_stats- assert "name" in result.field_stats- # Single item means constant fields- assert result.field_stats["id"].is_constant- assert result.field_stats["name"].is_constant-- def test_analyze_numeric_field_stats(self, analyzer):- """Numeric fields should have correct statistics computed."""- items = [- {"value": 10.0},- {"value": 20.0},- {"value": 30.0},- {"value": 40.0},- {"value": 50.0},- ]- result = analyzer.analyze_array(items)-- stats = result.field_stats["value"]- assert stats.field_type == "numeric"- assert stats.min_val == 10.0- assert stats.max_val == 50.0- assert stats.mean_val == 30.0- assert stats.variance is not None- assert stats.variance > 0-- def test_analyze_string_field_stats(self, analyzer):- """String fields should have correct statistics computed."""- items = [- {"name": "Alice"},- {"name": "Bob"},- {"name": "Alice"}, # Duplicate- {"name": "Charlie"},- {"name": "Alice"}, # Another duplicate- ]- result = analyzer.analyze_array(items)-- stats = result.field_stats["name"]- assert stats.field_type == "string"- assert stats.avg_length is not None- assert stats.top_values is not None- # Alice appears 3 times, should be top- assert stats.top_values[0][0] == "Alice"- assert stats.top_values[0][1] == 3-- def test_detect_time_series_pattern(self, analyzer):- """Time series data should be detected correctly."""- # Create data with timestamp and numeric variance- # Include anomaly to provide an importance signal for crushing- items = []- for i in range(40):- # Create variance-inducing data- value = 100.0 + (i * 2.0) # Steady increase with variance- if i == 20:- value = 999.0 # Anomaly provides importance signal- items.append(- {- "timestamp": f"2025-01-{(i % 28) + 1:02d}T12:00:00Z",- "value": value,- "metric": "cpu_usage",- }- )-- result = analyzer.analyze_array(items)-- # Pattern should be detected as time_series (timestamp + numeric variance)- assert result.detected_pattern == "time_series"- # With anomaly signal, strategy should allow crushing- assert result.recommended_strategy in [- CompressionStrategy.TIME_SERIES,- CompressionStrategy.SMART_SAMPLE,- ]-- def test_detect_time_series_pattern_with_change_points(self):- """Time series with clear change points should use TIME_SERIES strategy."""- # The change point detection threshold is variance_threshold * std- # To detect a change point, the before/after mean difference must exceed this- # With bimodal data, std is very high. We need a lower variance_threshold- # to reliably detect change points, OR the test should use a config- # with lower variance threshold.-- config = SmartCrusherConfig(- min_items_to_analyze=3,- variance_threshold=1.0, # Lower threshold to detect changes- )- analyzer = SmartAnalyzer(config)-- # Create data with clear step change- items = []- for i in range(40):- if i < 20:- value = 100.0 + (i * 0.5) # Values around 100-110- else:- value = 300.0 + ((i - 20) * 0.5) # Values around 300-310 (jump)- items.append(- {- "timestamp": f"2025-01-{(i % 28) + 1:02d}T12:00:00Z",- "value": value,- "metric": "cpu_usage",- }- )-- result = analyzer.analyze_array(items)-- assert result.detected_pattern == "time_series"- # With lower variance_threshold, change points should be detected- value_stats = result.field_stats.get("value")- assert value_stats is not None- # Even with low threshold, bimodal data has high std- # The test verifies the strategy selection logic- if len(value_stats.change_points) > 0:- assert result.recommended_strategy == CompressionStrategy.TIME_SERIES- else:- # If change points still not detected, strategy falls back- assert result.recommended_strategy in [- CompressionStrategy.TIME_SERIES,- CompressionStrategy.SMART_SAMPLE,- ]-- def test_detect_logs_pattern(self, analyzer):- """Log data should be detected correctly."""- # Use logs WITH errors to provide importance signal- items = generate_log_data(20, with_errors=True)- result = analyzer.analyze_array(items)-- # With structural detection, logs are detected as logs pattern- # but strategy depends on crushability analysis- assert result.detected_pattern in ["logs", "generic"]- # With error items providing signal, crushing can proceed- assert result.recommended_strategy in [- CompressionStrategy.CLUSTER_SAMPLE,- CompressionStrategy.SMART_SAMPLE,- CompressionStrategy.SKIP, # May still skip if other conditions met- ]-- def test_detect_search_results_pattern(self, analyzer):- """Search results with scores should be detected correctly."""- items = generate_search_results(20)- result = analyzer.analyze_array(items)-- assert result.detected_pattern == "search_results"- assert result.recommended_strategy == CompressionStrategy.TOP_N-- def test_detect_generic_pattern(self, analyzer):- """Generic data without special patterns should be detected."""- items = generate_generic_data(20)- result = analyzer.analyze_array(items)-- assert result.detected_pattern == "generic"- # With new crushability analysis: unique IDs + no importance signal = SKIP- # This is the safe behavior to avoid dropping important unique entities- assert result.recommended_strategy in [- CompressionStrategy.SMART_SAMPLE,- CompressionStrategy.SKIP, # More conservative when no signal present- ]-- def test_detect_change_points(self, analyzer):- """Change points should be detected in numeric data with variance."""- # Create data with clear change point- items = []- for i in range(30):- if i < 15:- value = 100.0 + (i * 0.1) # Low values- else:- value = 200.0 + ((i - 15) * 0.1) # High values after change- items.append({"timestamp": f"2025-01-{(i % 28) + 1:02d}", "metric": value})-- result = analyzer.analyze_array(items)-- # Should detect change point around index 15- metric_stats = result.field_stats.get("metric")- assert metric_stats is not None- assert metric_stats.change_points is not None- # Change points should be near the transition- if metric_stats.change_points:- assert any(10 <= cp <= 20 for cp in metric_stats.change_points)-- def test_constant_field_detection(self, analyzer):- """Constant fields should be identified."""- items = generate_generic_data(20, constant_field=True)- result = analyzer.analyze_array(items)-- # type field should be constant ("product")- type_stats = result.field_stats.get("type")- assert type_stats is not None- assert type_stats.is_constant- assert type_stats.constant_value == "product"-- # Constant fields should be in constant_fields dict- assert "type" in result.constant_fields- assert result.constant_fields["type"] == "product"---# =============================================================================-# TestSmartCrusher-# =============================================================================---class TestSmartCrusher:- """Tests for SmartCrusher transform."""-- def test_should_apply_below_threshold(self, tokenizer):- """Should not apply when tokens below min_tokens_to_crush."""- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": '{"status": "ok"}'},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=1000, # High threshold- )- crusher = SmartCrusher(config)-- assert not crusher.should_apply(messages, tokenizer)-- def test_should_apply_no_arrays(self, tokenizer):- """Should not apply when no crushable arrays present."""- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": '{"status": "ok", "value": 123}'},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- )- crusher = SmartCrusher(config)-- assert not crusher.should_apply(messages, tokenizer)-- def test_should_apply_small_array(self, tokenizer):- """Should not apply when array is below min_items_to_analyze."""- small_array = [{"id": i} for i in range(3)]- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(small_array)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=10, # Array too small- )- crusher = SmartCrusher(config)-- assert not crusher.should_apply(messages, tokenizer)-- def test_crush_time_series_keeps_change_points(self, tokenizer, default_config):- """Time series crushing should preserve items around change points."""- # Create data with clear change point AND an anomaly signal- items = []- for i in range(30):- if i < 15:- value = 100.0- else:- value = 200.0 # Jump at index 15- # Add anomaly to provide importance signal for crushing- if i == 25:- value = 999.0 # Extreme anomaly- items.append(- {- "timestamp": f"2025-01-{(i % 28) + 1:02d}T12:00:00Z",- "value": value,- }- )-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=15,- preserve_change_points=True,- )- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- # Parse result- tool_content = result.messages[1]["content"]- # Remove digest marker- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- # Should have fewer items than original- assert len(crushed) < len(items)-- # Should have items around change point (index ~15)- values = [item["value"] for item in crushed]- # Should have both low and high values (around change point)- assert 100.0 in values- assert 200.0 in values-- def test_crush_keeps_first_k_items(self, tokenizer):- """Crushing should always keep first K items (Safe V1 Recipe)."""- items = generate_generic_data(30)-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- )- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- # Parse result- tool_content = result.messages[1]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- # First 3 items should be preserved- crushed_ids = [item["id"] for item in crushed]- assert 0 in crushed_ids # First item- assert 1 in crushed_ids # Second item- assert 2 in crushed_ids # Third item-- def test_crush_keeps_last_k_items(self, tokenizer):- """Crushing should always keep last K items (Safe V1 Recipe)."""- items = generate_generic_data(30)-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- )- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- # Parse result- tool_content = result.messages[1]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- # Last 2 items should be preserved- crushed_ids = [item["id"] for item in crushed]- assert 28 in crushed_ids # Second to last- assert 29 in crushed_ids # Last item-- def test_crush_keeps_error_items(self, tokenizer):- """Crushing should always preserve error items (Safe V1 Recipe)."""- items = generate_generic_data(30)- # Add error items in the middle- items[10] = {"id": 10, "status": "error", "message": "Connection failed"}- items[20] = {"id": 20, "status": "failed", "exception": "TimeoutError"}-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- )- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- # Parse result- tool_content = result.messages[1]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- # Error items should be preserved- crushed_ids = [item["id"] for item in crushed]- assert 10 in crushed_ids # Error item- assert 20 in crushed_ids # Failed item-- def test_crush_keeps_anomalies(self, tokenizer):- """Crushing should preserve anomalous numeric items (> 2 std from mean)."""- items = []- for i in range(30):- value = 100.0 + (i * 0.1) # Normal range ~100-103- items.append({"id": i, "metric": value})-- # Add anomaly in the middle- items[15]["metric"] = 500.0 # Way above mean-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- variance_threshold=2.0,- )- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- # Parse result- tool_content = result.messages[1]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- # Anomaly should be preserved- crushed_ids = [item["id"] for item in crushed]- assert 15 in crushed_ids # Anomaly-- def test_crush_top_n_by_score(self, tokenizer):- """Search results should be crushed keeping top N by score."""- items = generate_search_results(30)-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- )- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- # Parse result- tool_content = result.messages[1]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- # Should have reduced items- assert len(crushed) < len(items)-- # Top scored items should be present- crushed_ids = [item["id"] for item in crushed]- assert "doc_0" in crushed_ids # Highest score- assert "doc_1" in crushed_ids # Second highest-- def test_schema_preserved(self, tokenizer):- """Output items should have same schema as input items."""- items = [- {- "id": i,- "name": f"Item {i}",- "nested": {"key": f"value_{i}"},- "tags": ["a", "b"],- }- for i in range(20)- ]-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- )- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- # Parse result- tool_content = result.messages[1]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- # Each item should have original schema- original_keys = set(items[0].keys())- for item in crushed:- assert set(item.keys()) == original_keys- assert "nested" in item- assert "key" in item["nested"]- assert "tags" in item- assert isinstance(item["tags"], list)-- def test_respects_max_items_after_crush(self, tokenizer):- """Output should respect max_items_after_crush limit."""- # Create data with importance signals (errors, anomalies) so crushing happens- items = []- for i in range(100):- item = {- "id": i,- "name": f"Item {i}",- "type": f"type_{i % 3}",- "value": 100.0,- }- # Add some errors to provide importance signal- if i in [10, 30, 50, 70, 90]:- item["error"] = f"Error at {i}"- # Add some anomalies- if i in [15, 45, 75]:- item["value"] = 99999.0- items.append(item)-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=15,- )- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- # Parse result- tool_content = result.messages[1]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- # With errors and anomalies, crushing should happen- # But critical items override max, so we may have more than 15- # The test verifies that crushing happened (fewer than original)- assert len(crushed) < 100, "Should compress the data"- # Errors must be preserved- error_count = sum(1 for x in crushed if x.get("error"))- assert error_count == 5, "All errors must be preserved"---# =============================================================================-# TestRelevanceScoring-# =============================================================================---class TestRelevanceScoring:- """Tests for relevance scoring in SmartCrusher."""-- def test_context_extraction_from_user_messages(self, tokenizer):- """Context should be extracted from user messages."""- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "user", "content": "Find user Alice in the database"},- {- "role": "assistant",- "content": None,- "tool_calls": [- {- "id": "call_1",- "type": "function",- "function": {"name": "search_users", "arguments": "{}"},- }- ],- },- {"role": "tool", "tool_call_id": "call_1", "content": "[]"},- ]-- config = SmartCrusherConfig(enabled=True, min_tokens_to_crush=0)- crusher = SmartCrusher(config)-- context = crusher._extract_context_from_messages(messages)-- assert "Alice" in context- assert "database" in context-- def test_context_extraction_from_tool_calls(self, tokenizer):- """Context should be extracted from tool call arguments."""- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "user", "content": "Search for it"},- {- "role": "assistant",- "content": None,- "tool_calls": [- {- "id": "call_1",- "type": "function",- "function": {- "name": "search_users",- "arguments": '{"query": "user_id=12345"}',- },- }- ],- },- {"role": "tool", "tool_call_id": "call_1", "content": "[]"},- ]-- config = SmartCrusherConfig(enabled=True, min_tokens_to_crush=0)- crusher = SmartCrusher(config)-- context = crusher._extract_context_from_messages(messages)-- assert "12345" in context-- def test_relevance_keeps_matching_items(self, tokenizer):- """Items matching user query should be preserved."""- # Create items where one matches the user query- items = [{"id": i, "name": f"User {i}", "email": f"user{i}@example.com"} for i in range(30)]- # Add special user Alice- items[15] = {"id": 15, "name": "Alice", "email": "[email protected]"}-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "user", "content": "Find user Alice"},- {- "role": "assistant",- "content": None,- "tool_calls": [- {- "id": "call_1",- "type": "function",- "function": {"name": "list_users", "arguments": "{}"},- }- ],- },- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- # Use BM25 scorer for deterministic testing- relevance_config = RelevanceScorerConfig(- tier="bm25",- relevance_threshold=0.1, # Lower threshold to catch matches- )- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- )- crusher = SmartCrusher(config, relevance_config=relevance_config)-- result = crusher.apply(messages, tokenizer)-- # Parse result- tool_content = result.messages[-1]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- # Alice should be preserved due to relevance matching- crushed_names = [item["name"] for item in crushed]- assert "Alice" in crushed_names-- def test_custom_scorer_injection(self, tokenizer):- """Custom scorer should be used when provided."""-- class MockScorer(RelevanceScorer):- """Mock scorer that always returns high score for items with id=5."""-- def score(self, item: str, context: str) -> RelevanceScore:- if '"id": 5' in item or '"id":5' in item:- return RelevanceScore(score=0.9, reason="mock high score")- return RelevanceScore(score=0.0, reason="mock low score")-- def score_batch(self, items: list[str], context: str) -> list[RelevanceScore]:- return [self.score(item, context) for item in items]-- items = generate_generic_data(30)-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "user", "content": "Find the special item"},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- )- crusher = SmartCrusher(config, scorer=MockScorer())-- result = crusher.apply(messages, tokenizer)-- # Parse result- tool_content = result.messages[-1]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- # Item with id=5 should be preserved due to mock scorer- crushed_ids = [item["id"] for item in crushed]- assert 5 in crushed_ids---# =============================================================================-# TestEdgeCases-# =============================================================================---class TestEdgeCases:- """Edge case tests for SmartCrusher."""-- def test_malformed_json_passthrough(self, tokenizer):- """Malformed JSON should pass through unchanged."""- malformed = "{ this is not valid JSON ["-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": malformed},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- )- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- # Content should be unchanged- assert result.messages[1]["content"] == malformed-- def test_nested_arrays(self, tokenizer):- """Nested arrays should be handled correctly."""- # Use with_signals=True to enable crushing with new statistical detection- nested_data = {- "results": generate_generic_data(20, with_signals=True),- "metadata": {- "inner_array": [{"x": i, "value": 100.0 if i != 7 else 99999.0} for i in range(15)],- },- }-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(nested_data)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- )- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- # Should be modified- tool_content = result.messages[1]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- # Results array should be crushed (with signals, crushing can happen)- assert len(crushed["results"]) < 20, "Results should be crushed"-- # Nested array should also be crushed if large enough- assert "metadata" in crushed- assert "inner_array" in crushed["metadata"]-- def test_anthropic_style_tool_results(self, tokenizer):- """Anthropic-style tool_result blocks should be handled."""- # Use with_signals=True to enable crushing with new statistical detection- items = generate_generic_data(20, with_signals=True)-- messages = [- {"role": "system", "content": "You are helpful."},- {- "role": "user",- "content": [- {"type": "text", "text": "What are the results?"},- {- "type": "tool_result",- "tool_use_id": "toolu_123",- "content": json.dumps(items),- },- ],- },- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- )- crusher = SmartCrusher(config)-- # Check should_apply works- assert crusher.should_apply(messages, tokenizer)-- result = crusher.apply(messages, tokenizer)-- # Tool result content should be crushed (with signals present)- tool_result_block = result.messages[1]["content"][1]- content = tool_result_block["content"]- json_part = content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- assert len(crushed) < len(items), "With signals, crushing should happen"-- def test_openai_style_tool_results(self, tokenizer):- """OpenAI-style tool messages should be handled."""- # Use with_signals=True to enable crushing with new statistical detection- items = generate_generic_data(20, with_signals=True)-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "user", "content": "Search for items"},- {- "role": "assistant",- "content": None,- "tool_calls": [- {- "id": "call_abc123",- "type": "function",- "function": {"name": "search_items", "arguments": "{}"},- }- ],- },- {- "role": "tool",- "tool_call_id": "call_abc123",- "content": json.dumps(items),- },- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- )- crusher = SmartCrusher(config)-- # Check should_apply works- assert crusher.should_apply(messages, tokenizer)-- result = crusher.apply(messages, tokenizer)-- # Tool content should be crushed- tool_content = result.messages[3]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- assert len(crushed) < len(items)- assert len(crushed) <= 10-- def test_empty_tool_content(self, tokenizer):- """Empty tool content should be handled gracefully."""- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": ""},- ]-- config = SmartCrusherConfig(enabled=True, min_tokens_to_crush=0)- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- # Should not crash, content unchanged- assert result.messages[1]["content"] == ""-- def test_non_dict_array_items(self, tokenizer):- """Arrays of non-dict items should be handled gracefully."""- items = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- )- crusher = SmartCrusher(config)-- # Should not crash, but won't crush non-dict arrays- result = crusher.apply(messages, tokenizer)-- # Primitive arrays are not crushed by SmartCrusher (it requires dict items)- # Content should pass through- assert result.messages is not None-- def test_mixed_null_values(self, tokenizer):- """Items with null values should be handled without crashing.-- With statistical detection, this data has unique IDs and no importance- signals, so it will be SKIPPED (not crushed) - which is correct behavior.- The test verifies that null values don't crash the analyzer.- """- items = [{"id": i, "value": None if i % 2 == 0 else i * 10} for i in range(20)]-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- )- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- # Should not crash - parsing should succeed- tool_content = result.messages[1]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- # With statistical detection, unique entities with no signals are SKIPPED- # This is correct conservative behavior - all 20 items preserved- assert len(crushed) == 20-- def test_unicode_content(self, tokenizer):- """Unicode content should be preserved."""- items = [- {"id": i, "name": f"Item {i} - \u4e2d\u6587 \u65e5\u672c\u8a9e \ud83d\ude80"}- for i in range(20)- ]-- messages = [- {"role": "system", "content": "You are helpful."},- {- "role": "tool",- "tool_call_id": "call_1",- "content": json.dumps(items, ensure_ascii=False),- },- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- )- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- tool_content = result.messages[1]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- # Unicode should be preserved- for item in crushed:- assert "\u4e2d\u6587" in item["name"]- assert "\u65e5\u672c\u8a9e" in item["name"]-- def test_digest_marker_added(self, tokenizer):- """Digest marker should be added to crushed content."""- items = generate_generic_data(30)-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- )- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- tool_content = result.messages[1]["content"]-- # Should have digest marker- assert "<headroom:tool_digest" in tool_content- assert "sha256=" in tool_content- assert len(result.markers_inserted) > 0-- def test_transforms_applied_tracking(self, tokenizer):- """Transforms applied should be tracked."""- items = generate_generic_data(30)-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- )- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- # Should track transforms- assert len(result.transforms_applied) > 0- assert any("smart" in t.lower() for t in result.transforms_applied)-- def test_token_reduction(self, tokenizer):- """Token count should be reduced after crushing."""- items = generate_generic_data(100)-- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- )- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- # Tokens should be reduced- assert result.tokens_after < result.tokens_before---# =============================================================================-# Integration Tests-# =============================================================================---class TestSmartCrusherIntegration:- """Integration tests for SmartCrusher with realistic scenarios."""-- def test_database_query_results(self, tokenizer):- """Simulate crushing database query results."""- # Simulate a database query returning many rows- items = [- {- "user_id": f"usr_{i:05d}",- "email": f"user{i}@example.com",- "created_at": f"2025-01-{(i % 28) + 1:02d}T00:00:00Z",- "status": "active" if i % 10 != 0 else "inactive",- "login_count": i * 5,- }- for i in range(100)- ]-- messages = [- {"role": "system", "content": "You are a database assistant."},- {"role": "user", "content": "Show me users with email containing 'user50'"},- {- "role": "assistant",- "tool_calls": [- {- "id": "call_1",- "type": "function",- "function": {"name": "query_users", "arguments": "{}"},- }- ],- },- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- relevance_config = RelevanceScorerConfig(tier="bm25", relevance_threshold=0.1)- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=15,- )- crusher = SmartCrusher(config, relevance_config=relevance_config)-- result = crusher.apply(messages, tokenizer)-- # Parse result- tool_content = result.messages[3]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- # Should be significantly reduced- assert len(crushed) <= 15- assert len(crushed) < len(items)-- # User 50 should be preserved due to relevance- user_ids = [item["user_id"] for item in crushed]- assert "usr_00050" in user_ids-- def test_api_search_results(self, tokenizer):- """Simulate crushing API search results."""- items = [- {- "id": f"result_{i}",- "title": f"Result {i}: {'Important Finding' if i < 5 else 'Regular Result'}",- "relevance_score": 0.95 - (i * 0.02),- "snippet": f"This is the snippet for result {i}...",- "url": f"https://example.com/doc/{i}",- }- for i in range(50)- ]-- messages = [- {"role": "system", "content": "You are a search assistant."},- {"role": "user", "content": "Search for important findings"},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=10,- )- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- # Parse result- tool_content = result.messages[2]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- # Should use TOP_N strategy- assert len(crushed) <= 12 # max + some buffer for first/last-- # Top results should be preserved- ids = [item["id"] for item in crushed]- assert "result_0" in ids- assert "result_1" in ids-- def test_monitoring_metrics(self, tokenizer):- """Simulate crushing monitoring/metrics data."""- items = []- for i in range(60):- # Normal CPU usage around 50%- cpu = 50.0 + (i * 0.1)- # Spike at index 30- if i == 30:- cpu = 95.0- items.append(- {- "timestamp": f"2025-01-06T{10 + (i // 60):02d}:{i % 60:02d}:00Z",- "cpu_percent": cpu,- "memory_percent": 60.0,- "host": "server-01",- }- )-- messages = [- {"role": "system", "content": "You are a monitoring assistant."},- {"role": "user", "content": "Show me CPU metrics"},- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},- ]-- config = SmartCrusherConfig(- enabled=True,- min_tokens_to_crush=0,- min_items_to_analyze=3,- max_items_after_crush=15,- preserve_change_points=True,- )- crusher = SmartCrusher(config)-- result = crusher.apply(messages, tokenizer)-- # Parse result- tool_content = result.messages[2]["content"]- json_part = tool_content.split("\n<headroom:")[0]- crushed = json.loads(json_part)-- # Should preserve the spike- cpu_values = [item["cpu_percent"] for item in crushed]- assert 95.0 in cpu_values # Spike should be preserveddiff --git a/tests/test_transforms/test_smart_crusher_bugs.py b/tests/test_transforms/test_smart_crusher_bugs.pyindex e09f2deee..3e7aa22e5 100644--- a/tests/test_transforms/test_smart_crusher_bugs.py+++ b/tests/test_transforms/test_smart_crusher_bugs.py@@ -20,6 +20,7 @@ from headroom.transforms.smart_crusher import SmartCrusherdef _make_crusher(max_items: int = 10, min_items: int = 3) -> SmartCrusher:+ """Build a SmartCrusher with deterministic small-K config for tests."""config = SmartCrusherConfig(enabled=True,min_items_to_analyze=min_items,@@ -30,102 +31,13 @@ def _make_crusher(max_items: int = 10, min_items: int = 3) -> SmartCrusher:return SmartCrusher(config=config)-# ----------------------------------------------------------------------------# Bug 1: Number array type mixing-# ------------------------------------------------------------------------------class TestNumberArraySchemaPreservation:- """_crush_number_array must return only original numeric values.-- Previously it prepended a stats summary string, producing- [string, int, int, ...] which violates the schema-preserving- guarantee and breaks type-aware JSON consumers.- """-- def test_crushed_number_array_contains_only_numbers(self) -> None:- """Every element of the crushed array must be int or float."""- crusher = _make_crusher(max_items=10)- numbers = list(range(50)) # 0..49, well above the n<=8 passthrough- crushed, strategy = crusher._crush_number_array(numbers)-- for i, item in enumerate(crushed):- assert isinstance(item, int | float), (- f"Item {i} is {type(item).__name__} = {item!r}, expected int/float. "- f"Schema-preserving guarantee violated."- )-- def test_crushed_number_array_subset_of_original(self) -> None:- """Every value in the crushed array must exist in the original."""- crusher = _make_crusher(max_items=10)- numbers = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]- crushed, _ = crusher._crush_number_array(numbers)-- original_set = set(numbers)- for item in crushed:- assert item in original_set, (- f"Value {item!r} not in original array — generated content detected"- )-- def test_stats_summary_in_strategy_not_in_array(self) -> None:- """Statistics should be communicated via strategy string, not array content."""- crusher = _make_crusher(max_items=5)- numbers = list(range(100))- crushed, strategy = crusher._crush_number_array(numbers)-- # Strategy should contain stats info- assert "number:" in strategy-- # Array should not contain any strings- strings_in_result = [x for x in crushed if isinstance(x, str)]- assert strings_in_result == [], f"Found string(s) in numeric array: {strings_in_result}"-- def test_number_array_passthrough_for_small(self) -> None:- """Arrays with n <= 8 should pass through unchanged."""- crusher = _make_crusher()- small = [1, 2, 3, 4, 5]- crushed, strategy = crusher._crush_number_array(small)- assert crushed == small- assert strategy == "number:passthrough"-- def test_number_array_preserves_outliers(self) -> None:- """Outlier values should be preserved in the crushed output."""- crusher = _make_crusher(max_items=10)- # Normal range + extreme outlier- numbers = [10] * 20 + [10000]- crushed, strategy = crusher._crush_number_array(numbers)- assert 10000 in crushed, "Outlier value 10000 was dropped"-- def test_number_array_preserves_boundaries(self) -> None:- """First and last values should always be kept."""- crusher = _make_crusher(max_items=5)- numbers = list(range(100))- crushed, strategy = crusher._crush_number_array(numbers)- assert crushed[0] == 0, "First value not preserved"- assert numbers[-1] in crushed, "Last value not preserved"-- def test_non_finite_passthrough(self) -> None:- """All-NaN/Inf arrays should return unchanged."""- crusher = _make_crusher()- nans = [float("nan")] * 10- crushed, strategy = crusher._crush_number_array(nans)- assert strategy == "number:no_finite"- assert len(crushed) == 10-- def test_full_crush_pipeline_number_array_types(self) -> None:- """End-to-end: crushing a JSON number array via the public API."""- crusher = _make_crusher(max_items=10)- content = json.dumps(list(range(50)))- result, was_modified, info = crusher._smart_crush_content(content)-- if was_modified:- parsed = json.loads(result)- assert isinstance(parsed, list)- for item in parsed:- assert isinstance(item, int | float), (- f"Public API returned non-numeric item {item!r} in number array"- )-+# Bug #1 (number array schema preservation) — invariant pinned by the+# Rust port (`crates/headroom-core/src/transforms/smart_crusher/crushers.rs::+# crush_number_array` + its unit tests) and the parity fixtures+# (`tests/parity/fixtures/smart_crusher/number_array_40_changepoint*`).+# The Python `_crush_number_array` helper that the previous tests+# probed was removed when the Python implementation was retired in+# Stage 3c.1b.# ---------------------------------------------------------------------------# Bug 2: Race condition on _current_field_semantics@@ -212,79 +124,12 @@ class TestRecursionDepthLimit:assert isinstance(parsed, list)-# ----------------------------------------------------------------------------# Stage 3c.1 lockstep bug fixes (#1 percentile, #2 sequential, #3 rare-status,-# #4 k-split). Each test pins the Python behavior post-fix; Rust has matching-# tests so parity fixtures byte-equal both languages.-# ------------------------------------------------------------------------------class TestStage3c1BugFixes:- """Bugs fixed in lockstep with the Rust port at Stage 3c.1."""-- # Bug #1 — percentile off-by-one (cosmetic, strategy string only).- def test_bug1_percentile_uses_linear_interpolation(self) -> None:- from headroom.transforms.smart_crusher import _percentile_linear-- # n=10 [10..100]: p25 index = 0.25 * 9 = 2.25 → 30*0.75 + 40*0.25 = 32.5- sorted_vals = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0]- assert _percentile_linear(sorted_vals, 0.25) == 32.5- assert _percentile_linear(sorted_vals, 0.75) == 77.5- # n=1 → return the single value.- assert _percentile_linear([42.0], 0.25) == 42.0- # n=0 → 0.0 (defensive).- assert _percentile_linear([], 0.25) == 0.0-- # Bug #2 — zero-padded string IDs misclassified as sequential.- def test_bug2_zero_padded_strings_not_sequential(self) -> None:- from headroom.transforms.smart_crusher import _detect_sequential_pattern-- # Pre-fix: int("001") → 1, so ["001",...,"005"] looked like 1..5- # and was classified as sequential. Post-fix: had_non_string_numeric- # stays False because every value came from a string → return False.- zero_padded = ["001", "002", "003", "004", "005"]- assert _detect_sequential_pattern(zero_padded, check_order=False) is False-- # Real ints (not strings) still classify as sequential.- real_ints = [1, 2, 3, 4, 5]- assert _detect_sequential_pattern(real_ints, check_order=False) is True-- # Bug #3 — rare-status detection cardinality cap.- def test_bug3_high_cardinality_pareto(self) -> None:- from headroom.transforms.smart_crusher import _detect_rare_status_values-- # 60×INFO + 25×WARN + 15 distinct error codes (cardinality 17).- # Pre-fix: 17 > 10 → field skipped → 0 outliers.- # Post-fix: top-2 covers 85% (60+25=85), K=2 ≤ 5, the 15 rare codes flagged.- items = []- for _ in range(60):- items.append({"code": "INFO"})- for _ in range(25):- items.append({"code": "WARN"})- for i in range(15):- items.append({"code": f"ERR_{i}"})- outliers = _detect_rare_status_values(items, common_fields={"code"})- assert len(outliers) == 15-- def test_bug3_uniform_distribution_no_outliers(self) -> None:- # 50 distinct values, 1 each → top-K never reaches 80% with K<=5.- # Field correctly identified as non-categorical.- from headroom.transforms.smart_crusher import _detect_rare_status_values-- items = [{"code": f"CAT_{i}"} for i in range(50)]- assert _detect_rare_status_values(items, common_fields={"code"}) == []-- # Bug #4 — k-split overshoot when k_total=1.- def test_bug4_k_split_no_overshoot_when_k_total_one(self) -> None:- from headroom import SmartCrusherConfig- from headroom.transforms.smart_crusher import SmartCrusher-- config = SmartCrusherConfig(min_items_to_analyze=1)- crusher = SmartCrusher(config=config)- # Force k_total=1 by passing a single-item list — the n<=8 fast- # path returns n=1, so k_total=1.- k_total, k_first, k_last, k_importance = crusher._compute_k_split(["only"], 1.0)- assert k_total == 1- assert k_first + k_last <= k_total, (- f"BUG #4: k_first={k_first} + k_last={k_last} must not exceed k_total={k_total}"- )+# Stage 3c.1 lockstep bug-fix tests previously lived here; they probed+# Python helpers (`_percentile_linear`, `_detect_sequential_pattern`,+# `_detect_rare_status_values`, `_compute_k_split`) that were removed+# along with the Python implementation in Stage 3c.1b. The Rust port+# pins the same invariants — see the `bug1_*` / `bug2_*` / `bug3_*` /+# `bug4_*` tests in `crates/headroom-core/src/transforms/smart_crusher/`+# (notably `crushers.rs` and `analyzer.rs`). Parity fixtures+# (`tests/parity/fixtures/smart_crusher/`) byte-compare the post-fix+# behavior across the language boundary.diff --git a/tests/test_transforms/test_universal_json_crush.py b/tests/test_transforms/test_universal_json_crush.pydeleted file mode 100644index 4d6eb16fa..000000000--- a/tests/test_transforms/test_universal_json_crush.py+++ /dev/null@@ -1,489 +0,0 @@-"""Tests for universal JSON compression (all array types).--Verifies that SmartCrusher handles arrays of dicts, strings, numbers,-mixed types, and nested arrays — with consistent safety guarantees-across all types.-"""--from __future__ import annotations--import json--import pytest--from headroom.transforms.smart_crusher import (- ArrayType,- SmartCrusher,- SmartCrusherConfig,- _classify_array,-)--# =====================================================================-# Fixtures-# =====================================================================---def crusher():- """SmartCrusher configured for testing."""- return SmartCrusher(- config=SmartCrusherConfig(- min_items_to_analyze=5,- min_tokens_to_crush=0, # Always crush- max_items_after_crush=15,- )- )---def crusher_large_k():- """SmartCrusher with higher max items for larger test arrays."""- return SmartCrusher(- config=SmartCrusherConfig(- min_items_to_analyze=5,- min_tokens_to_crush=0,- max_items_after_crush=50,- )- )---# =====================================================================-# Type Classification-# =====================================================================---class TestClassifyArray:- def test_dict_array(self):- assert _classify_array([{"a": 1}, {"b": 2}]) == ArrayType.DICT_ARRAY-- def test_string_array(self):- assert _classify_array(["hello", "world", "foo"]) == ArrayType.STRING_ARRAY-- def test_number_array_int(self):- assert _classify_array([1, 2, 3]) == ArrayType.NUMBER_ARRAY-- def test_number_array_float(self):- assert _classify_array([1.0, 2.5, 3.7]) == ArrayType.NUMBER_ARRAY-- def test_number_array_mixed_int_float(self):- assert _classify_array([1, 2.5, 3]) == ArrayType.NUMBER_ARRAY-- def test_bool_array(self):- assert _classify_array([True, False, True]) == ArrayType.BOOL_ARRAY-- def test_nested_array(self):- assert _classify_array([[1, 2], [3, 4]]) == ArrayType.NESTED_ARRAY-- def test_mixed_array(self):- assert _classify_array([{"a": 1}, "string", 42]) == ArrayType.MIXED_ARRAY-- def test_empty(self):- assert _classify_array([]) == ArrayType.EMPTY-- def test_single_dict(self):- assert _classify_array([{"key": "val"}]) == ArrayType.DICT_ARRAY-- def test_single_string(self):- assert _classify_array(["only"]) == ArrayType.STRING_ARRAY-- def test_none_values(self):- # Arrays with None mixed in are MIXED- assert _classify_array([1, None, 3]) == ArrayType.MIXED_ARRAY-- def test_bool_not_confused_with_int(self):- # Python's True/False are int subclasses — we handle this- assert _classify_array([True, False]) == ArrayType.BOOL_ARRAY- # But mixed bools and ints should be MIXED or NUMBER depending on impl- result = _classify_array([True, 42])- assert result in (ArrayType.MIXED_ARRAY, ArrayType.NUMBER_ARRAY)---# =====================================================================-# String Array Compression-# =====================================================================---class TestCrushStringArray:- def test_basic_compression(self, crusher):- strings = [f"item_{i}" for i in range(100)]- crushed, strategy = crusher._crush_string_array(strings)- assert len(crushed) < len(strings)- assert "string:adaptive" in strategy-- def test_errors_always_preserved(self, crusher):- strings = ["ok"] * 50 + ["error: connection timeout", "failed: auth denied"] + ["ok"] * 48- crushed, strategy = crusher._crush_string_array(strings)- assert any("error" in s for s in crushed)- assert any("failed" in s for s in crushed)-- def test_first_last_kept(self, crusher):- strings = [f"item_{i}" for i in range(50)]- crushed, strategy = crusher._crush_string_array(strings)- # First item always present- assert strings[0] in crushed- # Last item always present- assert strings[-1] in crushed-- def test_dedup_reduces_output(self, crusher):- # 95 identical + 5 unique- strings = ["repeated_value"] * 95 + [f"unique_{i}" for i in range(5)]- crushed, strategy = crusher._crush_string_array(strings)- # Should massively reduce — not keep 95 copies- assert len(crushed) < 20- # All 5 unique values should survive (they have high info value)- for i in range(5):- assert f"unique_{i}" in crushed-- def test_below_threshold_passthrough(self, crusher):- strings = ["a", "b", "c"] # Below min_items_to_analyze=5- # Direct method call — should passthrough since <= 8- crushed, strategy = crusher._crush_string_array(strings)- assert crushed == strings- assert "passthrough" in strategy-- def test_empty_strings_handled(self, crusher):- strings = [""] * 20- crushed, strategy = crusher._crush_string_array(strings)- # Should not crash- assert isinstance(crushed, list)-- def test_unicode_strings(self, crusher):- strings = [f"日本語テスト_{i}" for i in range(50)]- crushed, strategy = crusher._crush_string_array(strings)- assert len(crushed) < len(strings)- assert all(isinstance(s, str) for s in crushed)-- def test_length_anomalies_preserved(self, crusher_large_k):- # Most strings are short, one is very long- strings = ["short"] * 95 + ["x" * 10000] + ["short"] * 4- crushed, strategy = crusher_large_k._crush_string_array(strings)- assert any(len(s) > 1000 for s in crushed)---# =====================================================================-# Number Array Compression-# =====================================================================---class TestCrushNumberArray:- def test_basic_compression(self, crusher):- numbers = [42.0 + i * 0.1 for i in range(100)]- crushed, strategy = crusher._crush_number_array(numbers)- assert len(crushed) < len(numbers)- assert "number:adaptive" in strategy-- def test_stats_in_strategy_not_array(self, crusher):- numbers = list(range(100))- crushed, strategy = crusher._crush_number_array(numbers)- # Stats should be in the strategy string, not in the array- assert "min=" in strategy- assert "max=" in strategy- # Array should contain only numbers (schema-preserving)- for item in crushed:- assert isinstance(item, int | float)-- def test_outliers_preserved(self, crusher):- # Normal values around 50 with one extreme outlier- numbers = [50.0 + i * 0.01 for i in range(100)] + [999.9]- crushed, strategy = crusher._crush_number_array(numbers)- assert 999.9 in crushed- assert "outliers" in strategy-- def test_all_identical(self, crusher):- numbers = [42.0] * 100- crushed, strategy = crusher._crush_number_array(numbers)- # With all identical, should compress heavily- # Summary + a few representatives- numeric_values = [v for v in crushed if isinstance(v, int | float)]- assert all(v == 42.0 for v in numeric_values)-- def test_first_last_kept(self, crusher):- numbers = list(range(50))- crushed, strategy = crusher._crush_number_array(numbers)- numeric_values = [v for v in crushed if isinstance(v, int | float)]- assert 0 in numeric_values # First- assert 49 in numeric_values # Last-- def test_change_point_preserved(self, crusher_large_k):- # Stable at 10, then jumps to 100- numbers = [10.0] * 50 + [100.0] * 50- crushed, strategy = crusher_large_k._crush_number_array(numbers)- numeric_values = [v for v in crushed if isinstance(v, int | float)]- # Both 10.0 and 100.0 should be present- assert 10.0 in numeric_values- assert 100.0 in numeric_values-- def test_nan_inf_filtered(self, crusher):- numbers = [1.0, 2.0, float("nan"), float("inf"), 3.0] * 10- crushed, strategy = crusher._crush_number_array(numbers)- # Should not crash; stats in strategy based on finite values- assert "min=" in strategy- assert "max=" in strategy-- def test_integers_preserved_as_int(self, crusher):- numbers = list(range(50))- crushed, strategy = crusher._crush_number_array(numbers)- numeric_values = [v for v in crushed if isinstance(v, int | float)]- # Integers should remain integers (not converted to float)- assert any(isinstance(v, int) for v in numeric_values)-- def test_statistics_accuracy(self, crusher):- numbers = list(range(1, 101)) # 1 to 100- crushed, strategy = crusher._crush_number_array(numbers)- # Stats are in the strategy string- assert "min=1" in strategy- assert "max=100" in strategy- assert "mean=50.5" in strategy---# =====================================================================-# Mixed Array Compression-# =====================================================================---class TestCrushMixedArray:- def test_basic_compression(self, crusher_large_k):- mixed = [{"id": i} for i in range(30)] + [f"msg_{i}" for i in range(30)]- crushed, strategy = crusher_large_k._crush_mixed_array(mixed)- # With diversity-aware K, unique items may all be kept.- # Verify compression happened OR items are preserved due to high diversity.- assert len(crushed) <= len(mixed)- assert "mixed" in strategy-- def test_small_groups_kept(self, crusher):- # 50 dicts + 3 strings (below threshold)- mixed = [{"id": i} for i in range(50)] + ["rare_1", "rare_2", "rare_3"]- crushed, strategy = crusher._crush_mixed_array(mixed)- # All 3 rare strings should be kept (below min_items threshold)- assert "rare_1" in crushed- assert "rare_2" in crushed- assert "rare_3" in crushed-- def test_errors_across_types(self, crusher_large_k):- mixed = (- [{"status": "ok"}] * 30- + [{"status": "error: timeout"}]- + ["error: auth failed"]- + [f"ok_{i}" for i in range(30)]- )- crushed, strategy = crusher_large_k._crush_mixed_array(mixed)- crushed_str = json.dumps(crushed)- assert "error: timeout" in crushed_str- assert "error: auth failed" in crushed_str-- def test_original_order_preserved(self, crusher_large_k):- mixed = [{"id": i} for i in range(20)] + [f"str_{i}" for i in range(20)]- crushed, strategy = crusher_large_k._crush_mixed_array(mixed)- # Dicts should come before strings (original order)- first_str_idx = next(- (i for i, item in enumerate(crushed) if isinstance(item, str)), len(crushed)- )- last_dict_idx = max(- (i for i, item in enumerate(crushed) if isinstance(item, dict)), default=-1- )- assert last_dict_idx < first_str_idx-- def test_passthrough_small(self, crusher):- mixed = [1, "two", {"three": 3}]- crushed, strategy = crusher._crush_mixed_array(mixed)- assert crushed == mixed- assert "passthrough" in strategy---# =====================================================================-# Adaptive K-# =====================================================================---class TestAdaptiveK:- def test_scales_with_n(self, crusher):- """K grows sublinearly with collection size (or saturates at max_items)."""- small = [f"item_{i}" for i in range(20)]- large = [f"item_{i}" for i in range(500)]-- k_small = crusher._compute_k_split(small)[0]- k_large = crusher._compute_k_split(large)[0]-- # With diversity-aware K, both may saturate at max_items_after_crush- # for highly unique items. The key property: k never exceeds max_items.- assert k_large >= k_small- assert k_large <= crusher.config.max_items_after_crush-- def test_respects_max_items(self, crusher):- items = [f"item_{i}" for i in range(1000)]- k_total, _, _, _ = crusher._compute_k_split(items)- assert k_total <= crusher.config.max_items_after_crush-- def test_first_last_fractions(self, crusher):- items = [f"item_{i}" for i in range(100)]- k_total, k_first, k_last, k_importance = crusher._compute_k_split(items)- # First and last should be roughly the configured fractions- assert k_first >= 1- assert k_last >= 1- assert k_first + k_last + k_importance == k_total-- def test_homogeneous_vs_diverse(self, crusher_large_k):- """Homogeneous data should produce smaller K than diverse data."""- homogeneous = ["same value"] * 100- diverse = [f"unique_value_{i}_{'x' * (i * 10)}" for i in range(100)]-- k_homo = crusher_large_k._compute_k_split(homogeneous)[0]- k_diverse = crusher_large_k._compute_k_split(diverse)[0]-- # Diverse should need more items (or at least equal)- assert k_diverse >= k_homo---# =====================================================================-# Safety Guarantees (parametrized across types)-# =====================================================================---class TestSafetyGuarantees:- @pytest.mark.parametrize(- "items,item_type",- [- ([{"status": "ok"}] * 50 + [{"status": "error: timeout"}], "dict"),- (["ok"] * 50 + ["error: connection failed"], "string"),- (- [{"id": i} for i in range(30)]- + ["error: auth denied"]- + [f"ok_{i}" for i in range(30)],- "mixed",- ),- ],- ids=["dict_array", "string_array", "mixed_array"],- )- def test_errors_never_dropped(self, crusher_large_k, items, item_type):- """Error items must be preserved regardless of array type."""- if item_type == "dict":- crushed, _ = crusher_large_k._crush_string_array([json.dumps(i) for i in items])- crushed_text = " ".join(crushed)- elif item_type == "string":- crushed, _ = crusher_large_k._crush_string_array(items)- crushed_text = " ".join(crushed)- elif item_type == "mixed":- crushed, _ = crusher_large_k._crush_mixed_array(items)- crushed_text = json.dumps(crushed)-- assert "error" in crushed_text.lower()-- @pytest.mark.parametrize(- "items",- [- [f"item_{i}" for i in range(50)],- list(range(50)),- ],- ids=["string_array", "number_array"],- )- def test_first_last_always_present(self, crusher, items):- """First and last items must be present in output."""- if isinstance(items[0], str):- crushed, _ = crusher._crush_string_array(items)- assert items[0] in crushed- assert items[-1] in crushed- else:- crushed, _ = crusher._crush_number_array(items)- numeric = [v for v in crushed if isinstance(v, int | float)]- assert items[0] in numeric- assert items[-1] in numeric-- @pytest.mark.parametrize(- "items",- [- ["a", "b", "c"],- [1, 2, 3],- [{"x": 1}, "y", 2],- ],- ids=["string_below", "number_below", "mixed_below"],- )- def test_passthrough_below_min_items(self, crusher, items):- """Arrays below min_items_to_analyze pass through unchanged."""- if all(isinstance(i, str) for i in items):- crushed, strategy = crusher._crush_string_array(items)- elif all(isinstance(i, int | float) for i in items):- crushed, strategy = crusher._crush_number_array(items)- else:- crushed, strategy = crusher._crush_mixed_array(items)- assert "passthrough" in strategy---# =====================================================================-# Integration: Full pipeline-# =====================================================================---class TestFullPipelineIntegration:- """Test that new types work through the compress() API."""-- def test_string_array_via_compress(self):- from headroom import compress-- strings = [f"log line {i}: GET /api 200" for i in range(100)]- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "user", "content": "Show logs"},- {- "role": "assistant",- "content": None,- "tool_calls": [- {- "id": "call_1",- "type": "function",- "function": {"name": "get_logs", "arguments": "{}"},- }- ],- },- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(strings)},- ]- result = compress(messages)- assert result.tokens_saved > 0- assert result.compression_ratio > 0-- def test_number_array_via_compress(self):- from headroom import compress-- numbers = [42.0 + i * 0.1 for i in range(200)]- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "user", "content": "Show metrics"},- {- "role": "assistant",- "content": None,- "tool_calls": [- {- "id": "call_1",- "type": "function",- "function": {"name": "get_metrics", "arguments": "{}"},- }- ],- },- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(numbers)},- ]- result = compress(messages)- assert result.tokens_saved > 0-- def test_dict_array_unchanged(self):- """Verify dict arrays still work (regression test)."""- from headroom import compress-- data = [{"id": i, "name": f"user_{i}", "status": "active"} for i in range(100)]- messages = [- {"role": "system", "content": "You are helpful."},- {"role": "user", "content": "List users"},- {- "role": "assistant",- "content": None,- "tool_calls": [- {- "id": "call_1",- "type": "function",- "function": {"name": "list_users", "arguments": "{}"},- }- ],- },- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(data)},- ]- result = compress(messages)- assert result.tokens_saved > 0- assert result.compression_ratio > 0