Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(rust): retire python smart_crusher, ship rust-only via pyo3

chopratejasc765c53bf851
diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.py
index db8d284e8..dff901a43 100644
--- a/headroom/transforms/smart_crusher.py
+++ b/headroom/transforms/smart_crusher.py
@@ -1,3717 +1,329 @@
-"""Smart statistical tool output compression for Headroom SDK.
-
-This module provides intelligent JSON compression based on statistical analysis
-rather than fixed rules. It analyzes data patterns and applies optimal compression
-strategies to maximize token reduction while preserving important information.
-
-SCOPE: SmartCrusher handles JSON arrays of ANY type — dicts, strings, numbers,
-mixed types, and nested arrays. Non-JSON content (plain text, search results,
-logs, code, diffs) passes through UNCHANGED.
-
-TEXT COMPRESSION IS OPT-IN: For text-based content, Headroom provides standalone
-utilities that applications can use explicitly:
-- SearchCompressor: For grep/ripgrep output (file:line:content format)
-- LogCompressor: For build/test logs (pytest, npm, cargo output)
-- Kompress: For generic plain text (ML-based, requires [ml] extra)
-
-Applications should decide when and how to use text compression based on their
-specific needs. This design prevents lossy text compression from being applied
-automatically, which could lose important context in coding tasks.
-
-SCHEMA-PRESERVING: Output contains only items from the original array.
-No wrappers, no generated text, no metadata keys. This ensures downstream
-tools and parsers work unchanged.
-
-Supported JSON types:
-- Arrays of dicts: Full statistical analysis with adaptive K (Kneedle algorithm)
-- Arrays of strings: Dedup + adaptive sampling + error preservation
-- Arrays of numbers: Statistical summary + outlier/change-point preservation
-- Mixed-type arrays: Grouped by type, each group compressed independently
-- Flat objects (many keys): Key-level adaptive sampling
-- Nested objects: Recursive compression of inner arrays/objects
-
-Safety guarantees (consistent across ALL types):
-- First K, last K items always kept (K is adaptive, not hardcoded)
-- Error items (containing 'error', 'exception', 'failed', 'critical') never dropped
-- Anomalous numeric items (> 2 std from mean) always kept
-- Items around detected change points preserved
-- Items with high relevance score to user query (via RelevanceScorer)
-
-Key Features:
-- RelevanceScorer: ML-powered or BM25-based relevance matching (replaces regex)
-- Variance-based change point detection (preserve anomalies)
-- Error item detection (never lose error messages)
-- Pattern detection (time series, logs, search results)
-- Strategy selection based on data characteristics
-"""
-
-from __future__ import annotations
-
-import hashlib
-import json
-import logging
-import math
-import re
-import statistics
-import threading
-from collections import Counter
-from dataclasses import dataclass, field
-from enum import Enum
-from typing import Any
-
-from ..cache.compression_feedback import CompressionFeedback, get_compression_feedback
-from ..cache.compression_store import CompressionStore, get_compression_store
-from ..config import AnchorConfig, CCRConfig, RelevanceScorerConfig, TransformResult
-from ..relevance import RelevanceScorer, create_scorer
-from ..telemetry import TelemetryCollector, ToolSignature, get_telemetry_collector
-from ..telemetry.models import FieldSemantics
-from ..telemetry.toin import ToolIntelligenceNetwork, get_toin
-from ..tokenizer import Tokenizer
-from ..utils import (
- compute_short_hash,
- create_tool_digest_marker,
- deep_copy_messages,
- safe_json_dumps,
- safe_json_loads,
-)
-from .anchor_selector import AnchorSelector
-from .anchor_selector import DataPattern as AnchorDataPattern
-from .base import Transform
-from .error_detection import ERROR_KEYWORDS
-
-logger = logging.getLogger(__name__)
-
-# Legacy patterns for backwards compatibility (extract_query_anchors)
-_UUID_PATTERN = re.compile(
- r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
-)
-_NUMERIC_ID_PATTERN = re.compile(r"\b\d{4,}\b") # 4+ digit numbers (likely IDs)
-_HOSTNAME_PATTERN = re.compile(
- r"\b[a-zA-Z0-9][-a-zA-Z0-9]*\.[a-zA-Z0-9][-a-zA-Z0-9]*(?:\.[a-zA-Z]{2,})?\b"
-)
-_QUOTED_STRING_PATTERN = re.compile(r"['\"]([^'\"]{1,50})['\"]") # Short quoted strings
-_EMAIL_PATTERN = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b")
-
-# Temporal detection patterns (compiled once, used in SmartAnalyzer._detect_temporal_field)
-_ISO_DATETIME_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}")
-_ISO_DATE_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}$")
-
-
-def extract_query_anchors(text: str) -> set[str]:
- """Extract query anchors from user text (legacy regex-based method).
-
- DEPRECATED: Use RelevanceScorer.score_batch() for better semantic matching.
-
- Query anchors are identifiers or values that the user is likely searching for.
- When crushing tool outputs, items matching these anchors should be preserved.
-
- Extracts:
- - UUIDs (e.g., "550e8400-e29b-41d4-a716-446655440000")
- - Numeric IDs (4+ digits, e.g., "12345", "1001234")
- - Hostnames (e.g., "api.example.com", "server-01.prod")
- - Quoted strings (e.g., 'Alice', "error_code")
- - Email addresses (e.g., "[email protected]")
-
- Args:
- text: User message text to extract anchors from.
-
- Returns:
- Set of anchor strings (lowercased for case-insensitive matching).
- """
- anchors: set[str] = set()
-
- if not text:
- return anchors
-
- # UUIDs
- for match in _UUID_PATTERN.findall(text):
- anchors.add(match.lower())
-
- # Numeric IDs
- for match in _NUMERIC_ID_PATTERN.findall(text):
- anchors.add(match)
-
- # Hostnames
- for match in _HOSTNAME_PATTERN.findall(text):
- # Filter out common false positives
- if match.lower() not in ("e.g", "i.e", "etc."):
- anchors.add(match.lower())
-
- # Quoted strings
- for match in _QUOTED_STRING_PATTERN.findall(text):
- if len(match.strip()) >= 2: # Skip very short matches
- anchors.add(match.lower())
-
- # Email addresses
- for match in _EMAIL_PATTERN.findall(text):
- anchors.add(match.lower())
-
- return anchors
-
-
-def item_matches_anchors(item: dict, anchors: set[str]) -> bool:
- """Check if an item matches any query anchors (legacy method).
-
- DEPRECATED: Use RelevanceScorer for better matching.
-
- Args:
- item: Dictionary item from tool output.
- anchors: Set of anchor strings to match.
-
- Returns:
- True if any anchor is found in the item's string representation.
- """
- if not anchors:
- return False
-
- item_str = str(item).lower()
- return any(anchor in item_str for anchor in anchors)
-
-
-def _hash_field_name(field_name: str) -> str:
- """Hash a field name to match TOIN's anonymized preserve_fields.
-
- TOIN stores field names as SHA256[:8] hashes for privacy.
- This function produces the same hash format.
- """
- return hashlib.sha256(field_name.encode()).hexdigest()[:8]
-
-
-# Minimum chars for a text field to be worth compressing within an item
-_MIN_FIELD_CHARS_FOR_WITHIN = 200
-
-# Lazy-loaded compressor for within-item text compression (thread-safe)
-_within_compressor: Any = None
-_within_compressor_checked = False
-_within_compressor_lock = threading.Lock()
-
-
-def _get_within_compressor() -> Any:
- """Get a text compressor for within-item field compression.
-
- Returns Kompress if available (requires [ml] extra), else None.
- Thread-safe via double-checked locking.
- """
- global _within_compressor, _within_compressor_checked
- if not _within_compressor_checked:
- with _within_compressor_lock:
- if not _within_compressor_checked:
- try:
- from .kompress_compressor import KompressCompressor, is_kompress_available
-
- if is_kompress_available():
- _within_compressor = KompressCompressor()
- logger.debug("Within-item compression: using Kompress")
- except ImportError:
- pass
- _within_compressor_checked = True
- return _within_compressor
-
-
-def _compress_text_within_items(items: list[dict], context: str = "") -> list[dict]:
- """Compress long text fields WITHIN each item, keeping all items.
-
- Used when diversity is high (all items are unique) — instead of dropping
- items, compress the verbose text inside each one. Falls back to the
- original list unchanged if no compressor is available or no field is
- long enough to benefit.
-
- Args:
- items: JSON-parsed list of dicts.
- context: User query context for relevance-aware compression.
-
- Returns:
- Compressed items (new list) or the *same* ``items`` object if
- nothing was compressed (caller checks identity).
- """
- compressor = _get_within_compressor()
- if compressor is None:
- return items # No ML compressor available — pass through
-
- any_compressed = False
- result: list[dict] = []
-
- for item in items:
- new_item = dict(item) # Shallow copy
- item_changed = False
-
- for key, value in item.items():
- if not isinstance(value, str) or len(value) < _MIN_FIELD_CHARS_FOR_WITHIN:
- continue
-
- try:
- compressed = compressor.compress(value, context=context)
- if compressed.compressed and len(compressed.compressed) < len(value) * 0.9:
- new_item[key] = compressed.compressed
- item_changed = True
- except Exception:
- pass # Compression failed for this field — keep original
-
- result.append(new_item if item_changed else item)
- if item_changed:
- any_compressed = True
-
- return result if any_compressed else items
-
-
-def _get_preserve_field_values(
- item: dict,
- preserve_field_hashes: list[str],
-) -> list[tuple[str, Any]]:
- """Get values from item fields that match TOIN's preserve_field hashes.
-
- TOIN stores preserve_fields as hashed field names (SHA256[:8]).
- This function iterates over item fields, hashes each, and returns
- matching field names and values.
-
- Args:
- item: Dictionary item from tool output.
- preserve_field_hashes: List of SHA256[:8] hashed field names from TOIN.
-
- Returns:
- List of (field_name, value) tuples for fields that match.
- """
- if not preserve_field_hashes or not item:
- return []
-
- # Convert preserve_fields to set for O(1) lookup
- hash_set = set(preserve_field_hashes)
-
- matches = []
- for field_name, value in item.items():
- field_hash = _hash_field_name(field_name)
- if field_hash in hash_set:
- matches.append((field_name, value))
-
- return matches
-
-
-def _percentile_linear(sorted_values: list[float], q: float) -> float:
- """Linear-interpolation percentile, matching numpy's "linear" method.
-
- BUG #1 FIX (replaces integer-division indexing in _crush_number_array
- that was off by one for `len < 8`). Computes:
- index = q * (n - 1)
- if index is integer: return sorted_values[index]
- else: linear interpolate between floor and ceil
-
- Args:
- sorted_values: Pre-sorted ascending list of finite numbers.
- q: Quantile in [0, 1] (e.g. 0.25 for p25, 0.75 for p75).
-
- Returns:
- The interpolated quantile value. Empty input returns 0.0.
- """
- n = len(sorted_values)
- if n == 0:
- return 0.0
- if n == 1:
- return float(sorted_values[0])
- pos = q * (n - 1)
- lo = int(pos)
- hi = lo + 1 if lo + 1 < n else lo
- frac = pos - lo
- return sorted_values[lo] * (1 - frac) + sorted_values[hi] * frac
-
-
-def _item_has_preserve_field_match(
- item: dict,
- preserve_field_hashes: list[str],
- query_context: str,
-) -> bool:
- """Check if item has a preserve_field value that matches query context.
-
- Args:
- item: Dictionary item from tool output.
- preserve_field_hashes: List of SHA256[:8] hashed field names from TOIN.
- query_context: User's query to match against field values.
-
- Returns:
- True if any preserve_field value matches the query context.
- """
- if not query_context:
- return False
-
- query_lower = query_context.lower()
-
- for _field_name, value in _get_preserve_field_values(item, preserve_field_hashes):
- if value is not None:
- value_str = str(value).lower()
- if value_str in query_lower or query_lower in value_str:
- return True
-
- return False
-
-
-class CompressionStrategy(Enum):
- """Compression strategies based on data patterns."""
-
- NONE = "none" # No compression needed
- SKIP = "skip" # Explicitly skip - not safe to crush
- TIME_SERIES = "time_series" # Keep change points, summarize stable
- CLUSTER_SAMPLE = "cluster" # Dedupe similar items
- TOP_N = "top_n" # Keep highest scored items
- SMART_SAMPLE = "smart_sample" # Statistical sampling with constants
-
-
-class ArrayType(Enum):
- """JSON array element type classification."""
-
- DICT_ARRAY = "dict_array" # [{...}, {...}, ...]
- STRING_ARRAY = "string_array" # ["a", "b", "c", ...]
- NUMBER_ARRAY = "number_array" # [1, 2.5, 3, ...]
- BOOL_ARRAY = "bool_array" # [true, false, ...]
- NESTED_ARRAY = "nested_array" # [[...], [...], ...]
- MIXED_ARRAY = "mixed_array" # [{"a":1}, "str", 42, ...]
- EMPTY = "empty"
-
-
-def _classify_array(items: list) -> ArrayType:
- """Classify a JSON array by its element types.
-
- Uses set-of-types check on ALL elements (not sampling) to guarantee
- correct classification. Fast because type() is O(1).
- """
- if not items:
- return ArrayType.EMPTY
- # Note: bool is a subclass of int in Python, so check bool first
- types = set()
- has_bool = False
- for item in items:
- if isinstance(item, bool):
- has_bool = True
- types.add(type(item))
- if has_bool and types <= {bool, int}:
- # All bools (Python's True/False are int subclass)
- if all(isinstance(i, bool) for i in items):
- return ArrayType.BOOL_ARRAY
- if types == {dict}:
- return ArrayType.DICT_ARRAY
- if types == {str}:
- return ArrayType.STRING_ARRAY
- if types <= {int, float} and not has_bool:
- return ArrayType.NUMBER_ARRAY
- if types == {list}:
- return ArrayType.NESTED_ARRAY
- return ArrayType.MIXED_ARRAY
-
-
-# =====================================================================
-# STATISTICAL FIELD DETECTION (replaces hardcoded string patterns)
-# =====================================================================
-# Instead of matching field names like "id", "score", "error", we use
-# statistical and structural properties of the data to detect field types.
-
-
-def _is_uuid_format(value: str) -> bool:
- """Check if a string looks like a UUID (structural pattern)."""
- if not isinstance(value, str) or len(value) != 36:
- return False
- # UUID format: 8-4-4-4-12 hex chars
- parts = value.split("-")
- if len(parts) != 5:
- return False
- expected_lens = [8, 4, 4, 4, 12]
- for part, expected_len in zip(parts, expected_lens):
- if len(part) != expected_len:
- return False
- if not all(c in "0123456789abcdefABCDEF" for c in part):
- return False
- return True
-
-
-def _calculate_string_entropy(s: str) -> float:
- """Calculate Shannon entropy of a string, normalized to [0, 1].
-
- High entropy (>0.7) suggests random/ID-like content.
- Low entropy (<0.3) suggests repetitive/predictable content.
- """
- if not s or len(s) < 2:
- return 0.0
-
- # Count character frequencies
- freq: dict[str, int] = {}
- for c in s:
- freq[c] = freq.get(c, 0) + 1
-
- # Calculate entropy
- import math
-
- entropy = 0.0
- length = len(s)
- for count in freq.values():
- p = count / length
- if p > 0:
- entropy -= p * math.log2(p)
-
- # Normalize by max possible entropy for this length
- max_entropy = math.log2(min(len(freq), length))
- if max_entropy > 0:
- return entropy / max_entropy
- return 0.0
-
-
-def _detect_sequential_pattern(values: list[Any], check_order: bool = True) -> bool:
- """Detect if numeric values form a sequential pattern (like IDs: 1,2,3,...).
-
- Returns True if values appear to be auto-incrementing or sequential.
-
- Args:
- values: List of values to check.
- check_order: If True, also check if values are in ascending order in the array.
- Score fields are often sorted descending, while IDs are ascending.
- """
- if len(values) < 5:
- return False
-
- # BUG #2 FIX: track whether ANY value was already numeric (not
- # just a stringified number). If every "number" came from `int(s)`
- # of a string value, the field is almost certainly a categorical
- # string ID — possibly zero-padded — and should NOT be flagged
- # as sequential. Without this guard, `["001", "002", "003"]`
- # silently becomes `[1, 2, 3]` and gets misclassified.
- nums = []
- had_non_string_numeric = False
- for v in values:
- if isinstance(v, int | float) and not isinstance(v, bool):
- nums.append(v)
- had_non_string_numeric = True
- elif isinstance(v, str):
- try:
- nums.append(int(v))
- # Intentionally do NOT set had_non_string_numeric —
- # see fix doc above.
- except ValueError:
- pass
-
- if len(nums) < 5:
- return False
-
- # BUG #2 FIX: if every parseable value came from a string, the
- # field is categorical (zero-padded codes, alphanumeric IDs that
- # happen to be int-parseable, etc.). Don't classify as sequential.
- if not had_non_string_numeric:
- return False
-
- # Need at least 2 elements for pairwise comparison
- if len(nums) < 2:
- return False
-
- # Check if sorted values form a near-sequence
- sorted_nums = sorted(nums)
- diffs = [sorted_nums[i + 1] - sorted_nums[i] for i in range(len(sorted_nums) - 1)]
-
- if not diffs:
- return False
-
- # If most differences are 1 (or small constant), it's sequential
- avg_diff = sum(diffs) / len(diffs)
- if 0.5 <= avg_diff <= 2.0:
- # Check consistency - sequential IDs have consistent spacing
- consistent_count = sum(1 for d in diffs if 0.5 <= d <= 2.0)
- is_sequential = consistent_count / len(diffs) > 0.8
-
- # Additional check: IDs are typically in ASCENDING order in the array
- # Scores sorted by relevance are typically in DESCENDING order
- if check_order and is_sequential:
- # Check if original order is ascending (like IDs)
- ascending_count = sum(1 for i in range(len(nums) - 1) if nums[i] <= nums[i + 1])
- is_ascending = ascending_count / (len(nums) - 1) > 0.7
- return is_ascending # Only flag as sequential if ascending (ID-like)
-
- return is_sequential
-
- return False
-
-
-def _detect_id_field_statistically(stats: FieldStats, values: list[Any]) -> tuple[bool, float]:
- """Detect if a field is an ID field using statistical properties.
-
- Returns (is_id_field, confidence).
-
- ID fields have:
- - Very high uniqueness (>0.95)
- - Sequential numeric pattern OR UUID format OR high entropy strings
- """
- # Must have high uniqueness
- if stats.unique_ratio < 0.9:
- return False, 0.0
-
- confidence = 0.0
-
- # Check for UUID format (structural detection)
- if stats.field_type == "string":
- sample_values = [v for v in values[:20] if isinstance(v, str)]
- uuid_count = sum(1 for v in sample_values if _is_uuid_format(v))
- if sample_values and uuid_count / len(sample_values) > 0.8:
- return True, 0.95
-
- # Check for high entropy (random string IDs)
- if sample_values:
- avg_entropy = sum(_calculate_string_entropy(v) for v in sample_values) / len(
- sample_values
- )
- if avg_entropy > 0.7 and stats.unique_ratio > 0.95:
- confidence = 0.8
- return True, confidence
-
- # Check for sequential numeric pattern
- if stats.field_type == "numeric":
- if _detect_sequential_pattern(values) and stats.unique_ratio > 0.95:
- return True, 0.9
-
- # High uniqueness numeric with high range suggests ID
- if stats.min_val is not None and stats.max_val is not None:
- value_range = stats.max_val - stats.min_val
- if value_range > 0 and stats.unique_ratio > 0.95:
- return True, 0.85
-
- # Very high uniqueness alone is a signal (even without other patterns)
- if stats.unique_ratio > 0.98:
- return True, 0.7
-
- return False, 0.0
-
-
-def _detect_score_field_statistically(stats: FieldStats, items: list[dict]) -> tuple[bool, float]:
- """Detect if a field is a score/ranking field using statistical properties.
-
- Returns (is_score_field, confidence).
-
- Score fields have:
- - Numeric type
- - Bounded range (0-1, 0-10, 0-100, or similar)
- - NOT sequential (unlike IDs)
- - Often the data appears sorted by this field (descending)
- """
- if stats.field_type != "numeric":
- return False, 0.0
-
- if stats.min_val is None or stats.max_val is None:
- return False, 0.0
-
- confidence = 0.0
-
- # Check for bounded range typical of scores
- min_val, max_val = stats.min_val, stats.max_val
-
- # Common score ranges: [0,1], [0,10], [0,100], [-1,1], [0,5]
- is_bounded = False
- if 0 <= min_val <= 1 and 0 <= max_val <= 1: # [0,1] range
- is_bounded = True
- confidence += 0.4
- elif 0 <= min_val <= 10 and 0 <= max_val <= 10: # [0,10] range
- is_bounded = True
- confidence += 0.3
- elif 0 <= min_val <= 100 and 0 <= max_val <= 100: # [0,100] range
- is_bounded = True
- confidence += 0.25
- elif -1 <= min_val and max_val <= 1: # [-1,1] range
- is_bounded = True
- confidence += 0.35
-
- if not is_bounded:
- return False, 0.0
-
- # Should NOT be sequential (IDs are sequential, scores are not)
- sample_values = [item.get(stats.name) for item in items[:50] if stats.name in item]
- if _detect_sequential_pattern(sample_values):
- return False, 0.0
-
- # Check if data appears sorted by this field (descending = relevance sorted)
- # Filter out NaN/Inf which break comparisons
- values_in_order: list[float] = []
- for item in items:
- if stats.name in item:
- val = item.get(stats.name)
- if isinstance(val, int | float) and math.isfinite(val):
- values_in_order.append(float(val))
- if len(values_in_order) >= 5:
- # Check for descending sort
- num_pairs = len(values_in_order) - 1
- descending_count = sum(
- 1 for i in range(num_pairs) if values_in_order[i] >= values_in_order[i + 1]
- )
- if num_pairs > 0 and descending_count / num_pairs > 0.7:
- confidence += 0.3
-
- # Score fields often have floating point values
- # Filter out NaN/Inf which can't be converted to int
- float_count = sum(
- 1 for v in values_in_order[:20] if isinstance(v, float) and math.isfinite(v) and v != int(v)
- )
- if float_count > len(values_in_order[:20]) * 0.3:
- confidence += 0.1
-
- return confidence >= 0.4, min(confidence, 0.95)
-
-
-def _detect_structural_outliers(items: list[dict]) -> list[int]:
- """Detect items that are structural outliers (error-like items).
-
- Instead of looking for "error" keywords, we detect:
- 1. Items with extra fields that others don't have
- 2. Items with rare status/state values
- 3. Items with significantly different structure
-
- Returns indices of outlier items.
- """
- if len(items) < 5:
- return []
-
- outlier_indices: list[int] = []
-
- # 1. Detect items with extra fields
- # Find the "common" field set (fields present in >80% of items)
- field_counts: dict[str, int] = {}
- for item in items:
- if isinstance(item, dict):
- for key in item.keys():
- field_counts[key] = field_counts.get(key, 0) + 1
-
- n = len(items)
- common_fields = {k for k, v in field_counts.items() if v >= n * 0.8}
- rare_fields = {k for k, v in field_counts.items() if v < n * 0.2}
-
- for i, item in enumerate(items):
- if not isinstance(item, dict):
- continue
-
- item_fields = set(item.keys())
-
- # Has rare fields that most items don't have
- has_rare = bool(item_fields & rare_fields)
- if has_rare:
- outlier_indices.append(i)
- continue
-
- # 2. Detect rare status/state values
- # Find fields that look like status fields (low cardinality, categorical)
- status_outliers = _detect_rare_status_values(items, common_fields)
- outlier_indices.extend(status_outliers)
-
- return list(set(outlier_indices))
-
-
-def _detect_rare_status_values(items: list[dict], common_fields: set[str]) -> list[int]:
- """Detect items with rare values in status-like fields.
-
- A status field has low cardinality (few distinct values).
- If 95%+ have the same value, items with different values are interesting.
- """
- outlier_indices: list[int] = []
-
- # Find potential status fields (low cardinality)
- for field_name in common_fields:
- values = [
- item.get(field_name) for item in items if isinstance(item, dict) and field_name in item
- ]
-
- # Skip if too few values or non-hashable
- try:
- unique_values = {str(v) for v in values if v is not None}
- except Exception:
- continue
-
- # BUG #3 FIX: cardinality cap raised from 10 to 50 so that
- # higher-cardinality but Pareto-distributed fields (e.g.
- # 60 INFO + 25 WARN + 15 distinct error codes) still get
- # their rare values flagged. Above 50 distinct values, the
- # field is almost certainly an ID/free-form column, not a
- # status enum.
- if not (2 <= len(unique_values) <= 50):
- continue
-
- # Count value frequencies
- value_counts: dict[str, int] = {}
- for v in values:
- key = str(v) if v is not None else "__none__"
- value_counts[key] = value_counts.get(key, 0) + 1
-
- if not value_counts:
- continue
-
- total = len(values)
-
- # BUG #3 FIX: replace single-dominant check with Pareto top-K.
- # Sort frequencies descending; tiebreak by key ascending so the
- # outcome is deterministic when multiple values have the same
- # frequency. Find the smallest K such that top-K covers >=80%
- # of items.
- sorted_counts = sorted(value_counts.items(), key=lambda x: (-x[1], x[0]))
- threshold = math.ceil(total * 0.8)
- cumulative = 0
- top_k_values: set[str] = set()
- for value, count in sorted_counts:
- cumulative += count
- top_k_values.add(value)
- if cumulative >= threshold:
- break
-
- # K must be small (<=5) for any value to count as "rare".
- # Above this the distribution is too uniform to label a value
- # rare.
- if len(top_k_values) > 5:
- continue
-
- # Items NOT in top_k_values are outliers.
- for i, item in enumerate(items):
- if not isinstance(item, dict) or field_name not in item:
- continue
- item_value = str(item[field_name]) if item[field_name] is not None else "__none__"
- if item_value not in top_k_values:
- outlier_indices.append(i)
-
- return outlier_indices
-
-
-# Error keywords for PRESERVATION guarantee (not crushability detection)
-# This is for the quality guarantee: "ALL error items are ALWAYS preserved"
-# regardless of how common they are. Used in _prioritize_indices().
-# Centralized in error_detection module for consistency across transforms.
-_ERROR_KEYWORDS_FOR_PRESERVATION = ERROR_KEYWORDS
-
-
-def _detect_error_items_for_preservation(
- items: list[dict],
- item_strings: list[str] | None = None,
-) -> list[int]:
- """Detect items containing error keywords for PRESERVATION guarantee.
-
- This is NOT for crushability analysis - it's for ensuring ALL error items
- are retained during compression. The quality guarantee is that error items
- are NEVER dropped, even if errors are common in the dataset.
-
- Uses keywords because error semantics are well-defined across domains.
-
- Args:
- items: List of items to check.
- item_strings: Pre-computed JSON serializations to avoid redundant json.dumps.
- """
- error_indices: list[int] = []
-
- for i, item in enumerate(items):
- if not isinstance(item, dict):
- continue
-
- # Reuse cached serialization if available, otherwise serialize
- try:
- if item_strings is not None and i < len(item_strings):
- item_str = item_strings[i].lower()
- else:
- item_str = json.dumps(item).lower()
- except Exception:
- continue
-
- # Check if any error keyword is present
- for keyword in _ERROR_KEYWORDS_FOR_PRESERVATION:
- if keyword in item_str:
- error_indices.append(i)
- break
-
- return error_indices
-
-
-def _detect_items_by_learned_semantics(
- items: list[dict],
- field_semantics: dict[str, FieldSemantics],
-) -> list[int]:
- """Detect items with important values based on learned field semantics.
-
- This is the TOIN Evolution integration - uses learned field semantic types
- to identify items that should be preserved during compression.
-
- Key insight: Instead of hardcoded patterns, we learn from user behavior
- which field values are actually important (e.g., error indicators, rare
- status values, identifiers that get queried).
-
- Args:
- items: List of items to analyze.
- field_semantics: Learned field semantics from TOIN (field_hash -> FieldSemantics).
-
- Returns:
- List of indices for items containing important values.
- """
- if not field_semantics or not items:
- return []
-
- important_indices: list[int] = []
-
- # Build a quick lookup for field_hash -> FieldSemantics
- # Pre-filter to fields with sufficient confidence
- confident_semantics = {
- fh: fs
- for fh, fs in field_semantics.items()
- if fs.confidence >= 0.3 and fs.inferred_type != "unknown"
- }
-
- if not confident_semantics:
- return []
-
- # Pre-compute field name hashes to avoid redundant SHA256 per item
- _field_hash_cache: dict[str, str] = {}
-
- for i, item in enumerate(items):
- if not isinstance(item, dict):
- continue
-
- for field_name, value in item.items():
- # Hash the field name to match TOIN's format (cached per unique field name)
- if field_name not in _field_hash_cache:
- _field_hash_cache[field_name] = _hash_field_name(field_name)
- field_hash = _field_hash_cache[field_name]
-
- if field_hash not in confident_semantics:
- continue
-
- field_sem = confident_semantics[field_hash]
-
- # Hash the value to check importance
- if value is None:
- value_canonical = "null"
- elif isinstance(value, bool):
- value_canonical = "true" if value else "false"
- elif isinstance(value, int | float):
- value_canonical = str(value)
- elif isinstance(value, str):
- value_canonical = value
- elif isinstance(value, list | dict):
- try:
- value_canonical = json.dumps(value, sort_keys=True, default=str)
- except (TypeError, ValueError):
- value_canonical = str(value)
- else:
- value_canonical = str(value)
-
- value_hash = hashlib.sha256(value_canonical.encode()).hexdigest()[:8]
-
- # Check if this value is important based on learned semantics
- if field_sem.is_value_important(value_hash):
- important_indices.append(i)
- break # Only need to mark item once
-
- return important_indices
-
-
-@dataclass
-class CrushabilityAnalysis:
- """Analysis of whether an array is safe to crush.
-
- The key insight: if we don't have a reliable SIGNAL to determine
- which items are important, we should NOT crush at all.
-
- Signals include:
- - Score/rank fields (search results)
- - Error keywords (logs)
- - Numeric anomalies (metrics)
- - Low uniqueness (repetitive data where sampling is representative)
-
- High variability + No signal = DON'T CRUSH
- """
-
- crushable: bool
- confidence: float # 0.0 to 1.0
- reason: str
- signals_present: list[str] = field(default_factory=list)
- signals_absent: list[str] = field(default_factory=list)
-
- # Detailed metrics
- has_id_field: bool = False
- id_uniqueness: float = 0.0
- avg_string_uniqueness: float = 0.0
- has_score_field: bool = False
- error_item_count: int = 0
- anomaly_count: int = 0
-
-
-@dataclass
-class FieldStats:
- """Statistics for a single field across array items."""
-
- name: str
- field_type: str # "numeric", "string", "boolean", "object", "array", "null"
- count: int
- unique_count: int
- unique_ratio: float
- is_constant: bool
- constant_value: Any = None
-
- # Numeric-specific stats
- min_val: float | None = None
- max_val: float | None = None
- mean_val: float | None = None
- variance: float | None = None
- change_points: list[int] = field(default_factory=list)
-
- # String-specific stats
- avg_length: float | None = None
- top_values: list[tuple[str, int]] = field(default_factory=list)
-
-
-@dataclass
-class ArrayAnalysis:
- """Complete analysis of an array."""
-
- item_count: int
- field_stats: dict[str, FieldStats]
- detected_pattern: str # "time_series", "logs", "search_results", "generic"
- recommended_strategy: CompressionStrategy
- constant_fields: dict[str, Any]
- estimated_reduction: float
- crushability: CrushabilityAnalysis | None = None # Whether it's safe to crush
-
-
-@dataclass
-class CompressionPlan:
- """Plan for how to compress an array."""
-
- strategy: CompressionStrategy
- keep_indices: list[int] = field(default_factory=list)
- constant_fields: dict[str, Any] = field(default_factory=dict)
- summary_ranges: list[tuple[int, int, dict]] = field(default_factory=list)
- cluster_field: str | None = None
- sort_field: str | None = None
- keep_count: int = 10
-
-
-@dataclass
-class CrushResult:
- """Result from SmartCrusher.crush() method.
-
- Used by ContentRouter when routing JSON arrays to SmartCrusher.
- """
-
- compressed: str
- original: str
- was_modified: bool
- strategy: str = "passthrough"
-
-
-@dataclass
-class SmartCrusherConfig:
- """Configuration for smart crusher.
-
- SCHEMA-PRESERVING: Output contains only items from the original array.
- No wrappers, no generated text, no metadata keys.
- """
-
- enabled: bool = True
- min_items_to_analyze: int = 5 # Don't analyze tiny arrays
- min_tokens_to_crush: int = 200 # Only crush if > N tokens
- variance_threshold: float = 2.0 # Std devs for change point detection
- uniqueness_threshold: float = 0.1 # Below this = nearly constant
- similarity_threshold: float = 0.8 # For clustering similar strings
- max_items_after_crush: int = 15 # Target max items in output
- preserve_change_points: bool = True
- factor_out_constants: bool = False # Disabled - preserves original schema
- include_summaries: bool = False # Disabled - no generated text
-
- # Feedback loop integration
- use_feedback_hints: bool = True # Use learned patterns to adjust compression
-
- # LOW FIX #21: Make TOIN confidence threshold configurable
- # Minimum confidence required to apply TOIN recommendations
- toin_confidence_threshold: float = 0.5
-
- # Content deduplication - prevents wasting slots on identical items
- dedup_identical_items: bool = True
-
- # Adaptive K boundary allocation (fraction of total K for first/last items)
- first_fraction: float = 0.3 # 30% of K from start of array
- last_fraction: float = 0.15 # 15% of K from end of array
-
-
-class SmartAnalyzer:
- """Analyzes JSON arrays to determine optimal compression strategy."""
-
- def __init__(self, config: SmartCrusherConfig | None = None):
- self.config = config or SmartCrusherConfig()
-
- def analyze_array(self, items: list[dict]) -> ArrayAnalysis:
- """Perform complete statistical analysis of an array."""
- if not items or not isinstance(items[0], dict):
- return ArrayAnalysis(
- item_count=len(items) if items else 0,
- field_stats={},
- detected_pattern="generic",
- recommended_strategy=CompressionStrategy.NONE,
- constant_fields={},
- estimated_reduction=0.0,
- )
-
- # Analyze each field
- field_stats = {}
- all_keys: set[str] = set()
- for item in items:
- if isinstance(item, dict):
- all_keys.update(item.keys())
-
- # PARITY FIX (Stage 3c.1): iterate in sorted key order. Python's
- # set iteration is non-deterministic across PYTHONHASHSEED, so
- # downstream short-circuits (`_select_strategy` "message" lookup,
- # `_detect_pattern` first-score-field) become non-deterministic.
- # Rust uses BTreeMap which iterates ASCII-sorted; sorting here
- # locks both languages to the same iteration order so parity
- # fixtures byte-match.
- for key in sorted(all_keys):
- field_stats[key] = self._analyze_field(key, items)
-
- # Detect pattern
- pattern = self._detect_pattern(field_stats, items)
-
- # Extract constants
- constant_fields = {k: v.constant_value for k, v in field_stats.items() if v.is_constant}
-
- # CRITICAL: Analyze crushability BEFORE selecting strategy
- crushability = self.analyze_crushability(items, field_stats)
-
- # Select strategy (respects crushability)
- strategy = self._select_strategy(field_stats, pattern, len(items), crushability)
-
- # Estimate reduction (0 if not crushable)
- if strategy == CompressionStrategy.SKIP:
- reduction = 0.0
- else:
- reduction = self._estimate_reduction(field_stats, strategy, len(items))
-
- return ArrayAnalysis(
- item_count=len(items),
- field_stats=field_stats,
- detected_pattern=pattern,
- recommended_strategy=strategy,
- constant_fields=constant_fields,
- estimated_reduction=reduction,
- crushability=crushability,
- )
-
- def _analyze_field(self, key: str, items: list[dict]) -> FieldStats:
- """Analyze a single field across all items."""
- values = [item.get(key) for item in items if isinstance(item, dict)]
- non_null_values = [v for v in values if v is not None]
-
- if not non_null_values:
- return FieldStats(
- name=key,
- field_type="null",
- count=len(values),
- unique_count=0,
- unique_ratio=0.0,
- is_constant=True,
- constant_value=None,
- )
-
- # Determine type from first non-null value
- first_val = non_null_values[0]
- if isinstance(first_val, bool):
- field_type = "boolean"
- elif isinstance(first_val, int | float):
- field_type = "numeric"
- elif isinstance(first_val, str):
- field_type = "string"
- elif isinstance(first_val, dict):
- field_type = "object"
- elif isinstance(first_val, list):
- field_type = "array"
- else:
- field_type = "unknown"
-
- # Compute uniqueness
- str_values = [str(v) for v in values]
- unique_values = set(str_values)
- unique_count = len(unique_values)
- unique_ratio = unique_count / len(values) if values else 0
-
- # Check if constant
- is_constant = unique_count == 1
- constant_value = non_null_values[0] if is_constant else None
-
- stats = FieldStats(
- name=key,
- field_type=field_type,
- count=len(values),
- unique_count=unique_count,
- unique_ratio=unique_ratio,
- is_constant=is_constant,
- constant_value=constant_value,
- )
-
- # Numeric-specific analysis
- if field_type == "numeric":
- # Filter out NaN and Infinity which break statistics functions
- nums = [v for v in non_null_values if isinstance(v, int | float) and math.isfinite(v)]
- if nums:
- try:
- stats.min_val = min(nums)
- stats.max_val = max(nums)
- stats.mean_val = statistics.mean(nums)
- stats.variance = statistics.variance(nums) if len(nums) > 1 else 0
- stats.change_points = self._detect_change_points(nums)
- except (OverflowError, ValueError):
- # Extreme values that overflow - skip detailed statistics
- stats.min_val = None
- stats.max_val = None
- stats.mean_val = None
- stats.variance = 0
- stats.change_points = []
-
- # String-specific analysis
- elif field_type == "string":
- strs = [v for v in non_null_values if isinstance(v, str)]
- if strs:
- stats.avg_length = statistics.mean(len(s) for s in strs)
- stats.top_values = Counter(strs).most_common(5)
-
- return stats
-
- def _detect_change_points(self, values: list[float], window: int = 5) -> list[int]:
- """Detect indices where values change significantly."""
- if len(values) < window * 2:
- return []
-
- change_points = []
-
- # Calculate overall statistics
- overall_std = statistics.stdev(values) if len(values) > 1 else 0
- if overall_std == 0:
- return []
-
- threshold = self.config.variance_threshold * overall_std
-
- # Sliding window comparison
- for i in range(window, len(values) - window):
- before_mean = statistics.mean(values[i - window : i])
- after_mean = statistics.mean(values[i : i + window])
-
- if abs(after_mean - before_mean) > threshold:
- change_points.append(i)
-
- # Deduplicate nearby change points
- if change_points:
- deduped = [change_points[0]]
- for cp in change_points[1:]:
- if cp - deduped[-1] > window:
- deduped.append(cp)
- return deduped
-
- return []
-
- def _detect_pattern(self, field_stats: dict[str, FieldStats], items: list[dict]) -> str:
- """Detect the data pattern using STATISTICAL analysis (no hardcoded field names).
-
- Pattern detection:
- - TIME_SERIES: Has a temporal field (detected by value format) + numeric variance
- - LOGS: Has a high-cardinality string field + low-cardinality categorical field
- - SEARCH_RESULTS: Has a score-like field (bounded numeric, possibly sorted)
- - GENERIC: Default
- """
- # Check for time series pattern using STRUCTURAL detection
- has_timestamp = self._detect_temporal_field(field_stats, items)
-
- numeric_fields = [k for k, v in field_stats.items() if v.field_type == "numeric"]
- has_numeric_with_variance = any(
- (field_stats[k].variance is not None and (field_stats[k].variance or 0) > 0)
- for k in numeric_fields
- )
-
- if has_timestamp and has_numeric_with_variance:
- return "time_series"
-
- # Check for logs pattern using STATISTICAL detection
- # Logs have: high-cardinality string (message) + low-cardinality categorical (level)
- has_message_like = False
- has_level_like = False
-
- for _name, stats in field_stats.items():
- if stats.field_type == "string":
- # High-cardinality string = likely message field
- if stats.unique_ratio > 0.5 and stats.avg_length and stats.avg_length > 20:
- has_message_like = True
- # Low-cardinality string = likely level/status field
- elif stats.unique_ratio < 0.1 and 2 <= stats.unique_count <= 10:
- has_level_like = True
-
- if has_message_like and has_level_like:
- return "logs"
-
- # Check for search results pattern using STATISTICAL score detection
- for _name, stats in field_stats.items():
- is_score, confidence = _detect_score_field_statistically(stats, items)
- if is_score and confidence >= 0.5:
- return "search_results"
-
- return "generic"
-
- def _detect_temporal_field(self, field_stats: dict[str, FieldStats], items: list[dict]) -> bool:
- """Detect if any field contains temporal values (dates/timestamps).
-
- Uses STRUCTURAL detection based on value format, not field names.
- """
- # Check string fields for ISO 8601 patterns (module-level compiled)
- iso_datetime_pattern = _ISO_DATETIME_PATTERN
- iso_date_pattern = _ISO_DATE_PATTERN
-
- for name, stats in field_stats.items():
- if stats.field_type == "string":
- # Sample some values
- sample_values = [
- item.get(name) for item in items[:10] if isinstance(item.get(name), str)
- ]
- if sample_values:
- # Check if values look like dates/datetimes
- iso_count = sum(
- 1
- for v in sample_values
- if v is not None
- and (iso_datetime_pattern.match(v) or iso_date_pattern.match(v))
- )
- if iso_count / len(sample_values) > 0.5:
- return True
-
- # Check numeric fields for Unix timestamp range
- elif stats.field_type == "numeric":
- if stats.min_val and stats.max_val:
- # Unix timestamps (seconds): 1000000000 to 2000000000 (roughly 2001-2033)
- # Unix timestamps (milliseconds): 1000000000000 to 2000000000000
- is_unix_seconds = 1000000000 <= stats.min_val <= 2000000000
- is_unix_millis = 1000000000000 <= stats.min_val <= 2000000000000
- if is_unix_seconds or is_unix_millis:
- return True
-
- return False
-
- def analyze_crushability(
- self,
- items: list[dict],
- field_stats: dict[str, FieldStats],
- ) -> CrushabilityAnalysis:
- """Analyze whether it's SAFE to crush this array.
-
- The key insight: High variability + No importance signal = DON'T CRUSH.
-
- We use STATISTICAL detection (no hardcoded field names):
- 1. ID fields detected by uniqueness + sequential/UUID/entropy patterns
- 2. Score fields detected by bounded range + sorted order
- 3. Error items detected by structural outliers (rare fields, rare status values)
- 4. Numeric anomalies (importance signal)
- 5. Low uniqueness (safe to sample)
-
- Returns:
- CrushabilityAnalysis with decision and reasoning.
- """
- signals_present: list[str] = []
- signals_absent: list[str] = []
-
- # 1. Detect ID field STATISTICALLY (no hardcoded field names)
- id_field_name = None
- id_uniqueness = 0.0
- id_confidence = 0.0
- for name, stats in field_stats.items():
- values = [item.get(name) for item in items if isinstance(item, dict)]
- is_id, confidence = _detect_id_field_statistically(stats, values)
- if is_id and confidence > id_confidence:
- id_field_name = name
- id_uniqueness = stats.unique_ratio
- id_confidence = confidence
-
- has_id_field = id_field_name is not None and id_confidence >= 0.7
-
- # 2. Detect score/rank field STATISTICALLY (no hardcoded field names)
- has_score_field = False
- for name, stats in field_stats.items():
- is_score, confidence = _detect_score_field_statistically(stats, items)
- if is_score:
- has_score_field = True
- signals_present.append(f"score_field:{name}(conf={confidence:.2f})")
- break
- if not has_score_field:
- signals_absent.append("score_field")
-
- # 3. Detect error items via STRUCTURAL OUTLIERS (no hardcoded keywords)
- outlier_indices = _detect_structural_outliers(items)
- structural_outlier_count = len(outlier_indices)
-
- if structural_outlier_count > 0:
- signals_present.append(f"structural_outliers:{structural_outlier_count}")
- else:
- signals_absent.append("structural_outliers")
-
- # 3b. Also detect errors via keywords in content (for log/message-style data)
- # This catches errors that are in the content but not structural outliers
- # (e.g., Slack messages where error is in the text field)
- error_keyword_indices = _detect_error_items_for_preservation(items)
- keyword_error_count = len(error_keyword_indices)
-
- if keyword_error_count > 0 and structural_outlier_count == 0:
- signals_present.append(f"error_keywords:{keyword_error_count}")
-
- # Combined error count for crushability analysis
- error_count = max(structural_outlier_count, keyword_error_count)
-
- # 4. Count numeric anomalies (importance signal)
- anomaly_count = 0
- anomaly_indices: set[int] = set()
- for stats in field_stats.values():
- if stats.field_type == "numeric" and stats.mean_val is not None and stats.variance:
- std = stats.variance**0.5
- if std > 0:
- threshold = self.config.variance_threshold * std
- for i, item in enumerate(items):
- val = item.get(stats.name)
- if isinstance(val, int | float):
- if abs(val - stats.mean_val) > threshold:
- anomaly_indices.add(i)
-
- anomaly_count = len(anomaly_indices)
- if anomaly_count > 0:
- signals_present.append(f"anomalies:{anomaly_count}")
- else:
- signals_absent.append("anomalies")
-
- # 5. Compute average string uniqueness (EXCLUDING statistically-detected ID fields)
- string_stats = [
- s for s in field_stats.values() if s.field_type == "string" and s.name != id_field_name
- ]
- avg_string_uniqueness = (
- statistics.mean(s.unique_ratio for s in string_stats) if string_stats else 0.0
- )
-
- # Compute uniqueness of non-ID numeric fields
- non_id_numeric_stats = [
- s for s in field_stats.values() if s.field_type == "numeric" and s.name != id_field_name
- ]
- avg_non_id_numeric_uniqueness = (
- statistics.mean(s.unique_ratio for s in non_id_numeric_stats)
- if non_id_numeric_stats
- else 0.0
- )
-
- # Combined uniqueness metric (including ID fields)
- max_uniqueness = max(avg_string_uniqueness, id_uniqueness, 0.0)
-
- # Non-ID content uniqueness (for detecting repetitive content with unique IDs)
- non_id_content_uniqueness = max(avg_string_uniqueness, avg_non_id_numeric_uniqueness)
-
- # 6. Check for change points (importance signal for time series)
- has_change_points = any(
- stats.change_points for stats in field_stats.values() if stats.field_type == "numeric"
- )
- if has_change_points:
- signals_present.append("change_points")
-
- # DECISION LOGIC
- has_any_signal = len(signals_present) > 0
-
- # Case 0: Repetitive content with unique IDs
- # If all non-ID fields are nearly constant, data is safe to sample
- # even if there's a unique ID field (e.g., status="success" for all items)
- if non_id_content_uniqueness < 0.1 and has_id_field:
- signals_present.append("repetitive_content")
- return CrushabilityAnalysis(
- crushable=True,
- confidence=0.85,
- reason="repetitive_content_with_ids",
- signals_present=signals_present,
- signals_absent=signals_absent,
- has_id_field=has_id_field,
- id_uniqueness=id_uniqueness,
- avg_string_uniqueness=avg_string_uniqueness,
- has_score_field=has_score_field,
- error_item_count=error_count,
- anomaly_count=anomaly_count,
- )
-
- # Case 1: Low uniqueness - safe to sample (data is repetitive)
- if max_uniqueness < 0.3:
- return CrushabilityAnalysis(
- crushable=True,
- confidence=0.9,
- reason="low_uniqueness_safe_to_sample",
- signals_present=signals_present,
- signals_absent=signals_absent,
- has_id_field=has_id_field,
- id_uniqueness=id_uniqueness,
- avg_string_uniqueness=avg_string_uniqueness,
- has_score_field=has_score_field,
- error_item_count=error_count,
- anomaly_count=anomaly_count,
- )
-
- # Case 2: High uniqueness + ID field + NO signal = DON'T CRUSH
- # This is the critical case: DB results, file listings, user lists
- if has_id_field and max_uniqueness > 0.8 and not has_any_signal:
- return CrushabilityAnalysis(
- crushable=False,
- confidence=0.85,
- reason="unique_entities_no_signal",
- signals_present=signals_present,
- signals_absent=signals_absent,
- has_id_field=has_id_field,
- id_uniqueness=id_uniqueness,
- avg_string_uniqueness=avg_string_uniqueness,
- has_score_field=has_score_field,
- error_item_count=error_count,
- anomaly_count=anomaly_count,
- )
-
- # Case 3: High uniqueness + has signal = CRUSH using signal
- if max_uniqueness > 0.8 and has_any_signal:
- return CrushabilityAnalysis(
- crushable=True,
- confidence=0.7,
- reason="unique_entities_with_signal",
- signals_present=signals_present,
- signals_absent=signals_absent,
- has_id_field=has_id_field,
- id_uniqueness=id_uniqueness,
- avg_string_uniqueness=avg_string_uniqueness,
- has_score_field=has_score_field,
- error_item_count=error_count,
- anomaly_count=anomaly_count,
- )
-
- # Case 4: Medium uniqueness + no signal = be cautious, don't crush
- if not has_any_signal:
- return CrushabilityAnalysis(
- crushable=False,
- confidence=0.6,
- reason="medium_uniqueness_no_signal",
- signals_present=signals_present,
- signals_absent=signals_absent,
- has_id_field=has_id_field,
- id_uniqueness=id_uniqueness,
- avg_string_uniqueness=avg_string_uniqueness,
- has_score_field=has_score_field,
- error_item_count=error_count,
- anomaly_count=anomaly_count,
- )
-
- # Case 5: Medium uniqueness + has signal = crush with caution
- return CrushabilityAnalysis(
- crushable=True,
- confidence=0.5,
- reason="medium_uniqueness_with_signal",
- signals_present=signals_present,
- signals_absent=signals_absent,
- has_id_field=has_id_field,
- id_uniqueness=id_uniqueness,
- avg_string_uniqueness=avg_string_uniqueness,
- has_score_field=has_score_field,
- error_item_count=error_count,
- anomaly_count=anomaly_count,
- )
-
- def _select_strategy(
- self,
- field_stats: dict[str, FieldStats],
- pattern: str,
- item_count: int,
- crushability: CrushabilityAnalysis | None = None,
- ) -> CompressionStrategy:
- """Select optimal compression strategy based on analysis."""
- if item_count < self.config.min_items_to_analyze:
- return CompressionStrategy.NONE
-
- # CRITICAL: Check crushability first
- if crushability is not None and not crushability.crushable:
- return CompressionStrategy.SKIP
-
- if pattern == "time_series":
- # Check if there are change points worth preserving
- numeric_fields = [v for v in field_stats.values() if v.field_type == "numeric"]
- has_change_points = any(f.change_points for f in numeric_fields)
- if has_change_points:
- return CompressionStrategy.TIME_SERIES
-
- if pattern == "logs":
- # Check if messages are clusterable (low-medium uniqueness)
- message_field = next(
- (v for k, v in field_stats.items() if "message" in k.lower()), None
- )
- if message_field and message_field.unique_ratio < 0.5:
- return CompressionStrategy.CLUSTER_SAMPLE
-
- if pattern == "search_results":
- return CompressionStrategy.TOP_N
-
- # Default: smart sampling
- return CompressionStrategy.SMART_SAMPLE
-
- def _estimate_reduction(
- self, field_stats: dict[str, FieldStats], strategy: CompressionStrategy, item_count: int
- ) -> float:
- """Estimate token reduction ratio."""
- if strategy == CompressionStrategy.NONE:
- return 0.0
-
- # Count constant fields (will be factored out)
- constant_ratio = sum(1 for v in field_stats.values() if v.is_constant) / len(field_stats)
-
- # Estimate based on strategy
- base_reduction = {
- CompressionStrategy.TIME_SERIES: 0.7,
- CompressionStrategy.CLUSTER_SAMPLE: 0.8,
- CompressionStrategy.TOP_N: 0.6,
- CompressionStrategy.SMART_SAMPLE: 0.5,
- }.get(strategy, 0.3)
-
- # Adjust for constants
- reduction = base_reduction + (constant_ratio * 0.2)
-
- return min(reduction, 0.95)
-
-
-class SmartCrusher(Transform):
- """
- Intelligent tool output compression using statistical analysis.
-
- Unlike fixed-rule crushing, SmartCrusher:
- 1. Analyzes JSON structure and computes field statistics
- 2. Detects data patterns (time series, logs, search results)
- 3. Identifies constant fields to factor out
- 4. Finds change points in numeric data to preserve
- 5. Applies optimal compression strategy per data type
- 6. Uses RelevanceScorer for semantic matching of user queries
-
- This results in higher compression with lower information loss.
- """
-
- name = "smart_crusher"
-
- def __init__(
- self,
- config: SmartCrusherConfig | None = None,
- relevance_config: RelevanceScorerConfig | None = None,
- scorer: RelevanceScorer | None = None,
- ccr_config: CCRConfig | None = None,
- ):
- self.config = config or SmartCrusherConfig()
- self.analyzer = SmartAnalyzer(self.config)
-
- # CCR (Compress-Cache-Retrieve) configuration
- # When no ccr_config provided, default to caching enabled but markers disabled
- # This maintains backward compatibility - callers must opt-in to markers
- if ccr_config is None:
- self._ccr_config = CCRConfig(
- enabled=True, # Still cache for potential retrieval
- inject_retrieval_marker=False, # Don't break JSON parsing by default
- )
- else:
- self._ccr_config = ccr_config
- self._compression_store: CompressionStore | None = None
-
- # Feedback loop for learning compression patterns
- self._feedback: CompressionFeedback | None = None
-
- # CRITICAL FIX: Lock for thread-safe lazy initialization
- # Without this, multiple threads could call _get_* methods simultaneously
- # and potentially create redundant initialization calls.
- self._lazy_init_lock = threading.Lock()
-
- # Initialize relevance scorer
- if scorer is not None:
- self._scorer = scorer
- else:
- rel_config = relevance_config or RelevanceScorerConfig()
- # Build kwargs based on tier - BM25 params only apply to bm25 tier
- scorer_kwargs = {}
- if rel_config.tier == "bm25":
- scorer_kwargs = {"k1": rel_config.bm25_k1, "b": rel_config.bm25_b}
- elif rel_config.tier == "hybrid":
- scorer_kwargs = {
- "alpha": rel_config.hybrid_alpha,
- "adaptive": rel_config.adaptive_alpha,
- }
- self._scorer = create_scorer(tier=rel_config.tier, **scorer_kwargs)
- # Use threshold from config, or default from RelevanceScorerConfig
- rel_cfg = relevance_config or RelevanceScorerConfig()
- self._relevance_threshold = rel_cfg.relevance_threshold
-
- # Initialize AnchorSelector for dynamic position-based preservation
- anchor_config = self.config.anchor if hasattr(self.config, "anchor") else AnchorConfig()
- self._anchor_selector = AnchorSelector(anchor_config)
-
- # NOTE: Error detection now uses structural outlier detection (_detect_structural_outliers)
- # instead of hardcoded keywords. This scales to any data domain.
-
- def _map_to_anchor_pattern(self, strategy: CompressionStrategy) -> AnchorDataPattern:
- """Map SmartCrusher compression strategy to AnchorSelector data pattern.
-
- Args:
- strategy: The detected compression strategy.
-
- Returns:
- Corresponding AnchorDataPattern for anchor selection.
- """
- return {
- CompressionStrategy.TIME_SERIES: AnchorDataPattern.TIME_SERIES,
- CompressionStrategy.TOP_N: AnchorDataPattern.SEARCH_RESULTS,
- CompressionStrategy.CLUSTER_SAMPLE: AnchorDataPattern.LOGS,
- CompressionStrategy.SMART_SAMPLE: AnchorDataPattern.GENERIC,
- }.get(strategy, AnchorDataPattern.GENERIC)
-
- def crush(self, content: str, query: str = "", bias: float = 1.0) -> CrushResult:
- """Crush content string directly (for use by ContentRouter).
-
- This is a simplified interface for compressing a single content string,
- used by ContentRouter when routing JSON arrays to SmartCrusher.
-
- Args:
- content: JSON string content to compress.
- query: Query context for relevance-based compression.
- bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer).
-
- Returns:
- CrushResult with compressed content and metadata.
- """
- compressed, was_modified, analysis_info = self._smart_crush_content(
- content, query_context=query, bias=bias
- )
- return CrushResult(
- compressed=compressed,
- original=content,
- was_modified=was_modified,
- strategy=analysis_info or "passthrough",
- )
-
- def _get_compression_store(self) -> CompressionStore:
- """Get the compression store for CCR (lazy initialization).
-
- CRITICAL FIX: Thread-safe double-checked locking pattern.
- """
- if self._compression_store is None:
- with self._lazy_init_lock:
- # Double-check after acquiring lock
- if self._compression_store is None:
- self._compression_store = get_compression_store(
- max_entries=self._ccr_config.store_max_entries,
- default_ttl=self._ccr_config.store_ttl_seconds,
- )
- return self._compression_store
-
- def _get_feedback(self) -> CompressionFeedback:
- """Get the feedback analyzer (lazy initialization).
-
- CRITICAL FIX: Thread-safe double-checked locking pattern.
- """
- if self._feedback is None:
- with self._lazy_init_lock:
- if self._feedback is None:
- self._feedback = get_compression_feedback()
- return self._feedback
-
- def _get_telemetry(self) -> TelemetryCollector:
- """Get the telemetry collector (lazy initialization).
-
- CRITICAL FIX: Thread-safe double-checked locking pattern.
- """
- # Use getattr to avoid hasattr race condition
- if getattr(self, "_telemetry", None) is None:
- with self._lazy_init_lock:
- if getattr(self, "_telemetry", None) is None:
- self._telemetry = get_telemetry_collector()
- return self._telemetry
-
- def _get_toin(self) -> ToolIntelligenceNetwork:
- """Get the TOIN instance (lazy initialization).
-
- CRITICAL FIX: Thread-safe double-checked locking pattern.
- """
- # Use getattr to avoid hasattr race condition
- if getattr(self, "_toin", None) is None:
- with self._lazy_init_lock:
- if getattr(self, "_toin", None) is None:
- self._toin = get_toin()
- return self._toin
-
- def _record_telemetry(
- self,
- items: list[dict],
- result: list,
- analysis: ArrayAnalysis,
- plan: CompressionPlan,
- tool_name: str | None = None,
- ) -> None:
- """Record compression telemetry for the data flywheel.
-
- This collects anonymized statistics about compression patterns to
- enable cross-user learning and improve compression over time.
-
- Privacy guarantees:
- - No actual data values are stored
- - Tool names can be hashed
- - Only structural patterns are captured
- """
- try:
- telemetry = self._get_telemetry()
-
- # Calculate what was kept
- kept_first_n = sum(1 for i in plan.keep_indices if i < 3)
- kept_last_n = sum(1 for i in plan.keep_indices if i >= len(items) - 2)
-
- # Count error items in result
- error_indices = set(_detect_error_items_for_preservation(items))
- kept_errors = sum(1 for i in plan.keep_indices if i in error_indices)
-
- # Count anomalies (approximate from change points)
- anomaly_count = 0
- for stats in analysis.field_stats.values():
- if stats.change_points:
- anomaly_count += len(stats.change_points)
- kept_anomalies = min(anomaly_count, len(plan.keep_indices))
-
- # Crushability info
- crushability_score = None
- crushability_reason = None
- if analysis.crushability:
- crushability_score = analysis.crushability.confidence
- crushability_reason = analysis.crushability.reason
-
- # Record the event
- telemetry.record_compression(
- items=items[:100], # Sample for structure analysis
- original_count=len(items),
- compressed_count=len(result),
- original_tokens=0, # Not available here
- compressed_tokens=0, # Not available here
- strategy=analysis.recommended_strategy.value,
- tool_name=tool_name,
- strategy_reason=analysis.detected_pattern,
- crushability_score=crushability_score,
- crushability_reason=crushability_reason,
- kept_first_n=kept_first_n,
- kept_last_n=kept_last_n,
- kept_errors=kept_errors,
- kept_anomalies=kept_anomalies,
- kept_by_relevance=0, # Would need to track separately
- kept_by_score=0, # Would need to track separately
- )
- except Exception:
- # Telemetry should never break compression
- pass
-
- def _deduplicate_indices_by_content(
- self,
- keep_indices: set[int],
- items: list[dict],
- ) -> set[int]:
- """Deduplicate indices by content hash, preferring lower indices.
-
- When multiple indices contain identical content, only the lowest index
- is kept. This maximizes information density in the compressed output.
-
- Enterprise considerations:
- - Thread-safe: No shared state modified
- - Performance: O(n) single pass with hash map
- - Memory: O(n) for hash storage (16-char hashes)
- - Fault-tolerant: Serialization errors preserve the item
-
- Args:
- keep_indices: Set of indices to deduplicate.
- items: The items array for content lookup.
-
- Returns:
- Deduplicated set of indices (may be smaller than input).
- """
- if not keep_indices:
- return keep_indices
-
- # Track first occurrence of each content hash
- # Using dict[hash -> lowest_index] for O(1) lookup
- seen_hashes: dict[str, int] = {}
- duplicates_removed = 0
-
- # Process in sorted order to ensure deterministic "lowest index wins"
- for idx in sorted(keep_indices):
- # Bounds check
- if idx < 0 or idx >= len(items):
- continue
-
- item = items[idx]
-
- # Compute content hash
- try:
- if isinstance(item, dict):
- # Canonical JSON serialization for consistent hashing
- content = json.dumps(item, sort_keys=True, default=str)
- else:
- # Non-dict items: use string representation
- content = str(item)
- item_hash = hashlib.md5(content.encode()).hexdigest()[:16] # nosec B324
- except (TypeError, ValueError, RecursionError) as e:
- # Serialization failed - keep the item (fail-safe)
- logger.debug("Dedup hash failed for item at index %d: %s. Keeping item.", idx, e)
- # Use index as unique "hash" to ensure item is kept
- item_hash = f"__idx_{idx}__"
-
- # First occurrence wins
- if item_hash not in seen_hashes:
- seen_hashes[item_hash] = idx
- else:
- duplicates_removed += 1
-
- # Log deduplication stats for observability
- if duplicates_removed > 0:
- logger.debug(
- "Content deduplication removed %d duplicate items from %d candidates "
- "(%.1f%% reduction). Unique items: %d",
- duplicates_removed,
- len(keep_indices),
- 100 * duplicates_removed / len(keep_indices),
- len(seen_hashes),
- )
-
- return set(seen_hashes.values())
-
- def _fill_remaining_slots(
- self,
- keep_indices: set[int],
- items: list[dict],
- n: int,
- effective_max: int,
- ) -> set[int]:
- """Fill remaining slots with unique items when under budget.
-
- When deduplication reduces keep_indices below effective_max, this method
- fills the remaining slots with diverse items from the array. It:
- 1. Computes content hashes for already-kept items
- 2. Scans array for items with unique content not yet kept
- 3. Distributes new items evenly across the array for coverage
-
- Enterprise considerations:
- - Thread-safe: No shared state modified
- - Performance: O(n) for hash computation, O(n) for filling
- - Memory: O(k) where k = len(keep_indices) for hash storage
- - Deterministic: Same input always produces same output
-
- Args:
- keep_indices: Current set of indices to keep (already deduplicated).
- items: The full items array.
- n: Total number of items.
- effective_max: Target maximum items.
-
- Returns:
- Updated set of indices, filled up to effective_max with unique items.
- """
- remaining_slots = effective_max - len(keep_indices)
- if remaining_slots <= 0:
- return keep_indices
-
- # Build set of content hashes for items we're already keeping
- seen_hashes: set[str] = set()
- for idx in keep_indices:
- if 0 <= idx < n:
- item = items[idx]
- try:
- if isinstance(item, dict):
- content = json.dumps(item, sort_keys=True, default=str)
- else:
- content = str(item)
- seen_hashes.add(hashlib.md5(content.encode()).hexdigest()[:16]) # nosec B324
- except (TypeError, ValueError, RecursionError):
- pass # Skip hash computation failures
-
- # Find candidate indices not in keep_indices
- candidates = [i for i in range(n) if i not in keep_indices]
- if not candidates:
- return keep_indices
-
- # Distribute selection evenly across the array for coverage
- # Use stride-based sampling to avoid clustering
- result = keep_indices.copy()
- added = 0
-
- # Calculate step size for even distribution
- step = max(1, len(candidates) // (remaining_slots + 1))
-
- # First pass: evenly distributed unique items
- for start_offset in range(step):
- if added >= remaining_slots:
- break
- for i in range(start_offset, len(candidates), step):
- if added >= remaining_slots:
- break
- idx = candidates[i]
- item = items[idx]
-
- # Check if this item's content is unique
- try:
- if isinstance(item, dict):
- content = json.dumps(item, sort_keys=True, default=str)
- else:
- content = str(item)
- item_hash = hashlib.md5(content.encode()).hexdigest()[:16] # nosec B324
- except (TypeError, ValueError, RecursionError):
- # Hash failure - use index as unique hash (fail-safe)
- item_hash = f"__idx_{idx}__"
-
- if item_hash not in seen_hashes:
- result.add(idx)
- seen_hashes.add(item_hash)
- added += 1
-
- if added > 0:
- logger.debug(
- "Filled %d remaining slots (from %d to %d items) after deduplication",
- added,
- len(keep_indices),
- len(result),
- )
-
- return result
-
- def _prioritize_indices(
- self,
- keep_indices: set[int],
- items: list[dict],
- n: int,
- analysis: ArrayAnalysis | None = None,
- max_items: int | None = None,
- field_semantics: dict[str, FieldSemantics] | None = None,
- ) -> set[int]:
- """Prioritize indices when we exceed max_items, ALWAYS keeping critical items.
-
- Priority order:
- 1. ALL error items (non-negotiable) - items with error keywords
- 2. ALL structural outliers (non-negotiable) - items with rare fields/status values
- 3. ALL numeric anomalies (non-negotiable) - e.g., unusual values like 999999
- 4. ALL items with important values (learned) - TOIN field semantics
- 5. First 3 items (context)
- 6. Last 2 items (context)
- 7. Other important items by index order
-
- Uses BOTH keyword detection (for preservation guarantee) AND statistical detection,
- PLUS learned field semantics from TOIN for zero-latency signal detection.
-
- HIGH FIX: Note that this function may return MORE items than effective_max
- when critical items (errors, outliers, anomalies) exceed the limit. This is
- intentional to preserve the quality guarantee. A warning is logged when this
- happens to help diagnose cases where compression is less effective than expected.
-
- Args:
- keep_indices: Initial set of indices to keep.
- items: The items being compressed.
- n: Total number of items.
- analysis: Optional analysis results for anomaly detection.
- max_items: Thread-safe max items limit (defaults to config value).
- field_semantics: Optional learned field semantics from TOIN.
-
- Returns:
- Set of indices to keep (may exceed max_items if critical items require it).
- """
- # Use provided max_items or fall back to config
- effective_max = max_items if max_items is not None else self.config.max_items_after_crush
-
- # === ENTERPRISE FIX: Content-based deduplication ===
- # Multiple preservation mechanisms (anchors, anomalies, outliers, etc.) can add
- # the same item multiple times by index. More critically, different INDICES can
- # contain IDENTICAL content (e.g., 10 identical status messages at indices 0-9).
- # Without deduplication, we waste slots on redundant information.
- #
- # This deduplication runs FIRST, before any other logic, to ensure:
- # 1. Budget calculations work with unique items only
- # 2. Critical item detection doesn't double-count
- # 3. Final output maximizes information density
- #
- # Performance: O(n) where n = len(keep_indices), single pass with hash map
- # Memory: O(n) for hash storage, hashes are 16 chars each
- if self.config.dedup_identical_items:
- keep_indices = self._deduplicate_indices_by_content(keep_indices, items)
-
- # === ENTERPRISE FIX: Fill up to max_items when under budget ===
- # After deduplication, we may have fewer unique items than max_items.
- # Instead of returning a sparse result, fill remaining slots with
- # diverse items from the array. This maximizes information density.
- if len(keep_indices) < effective_max and len(keep_indices) < n:
- keep_indices = self._fill_remaining_slots(keep_indices, items, n, effective_max)
-
- if len(keep_indices) <= effective_max:
- return keep_indices
-
- # Use provided field_semantics or fall back to thread-local (set by _crush_array)
- effective_field_semantics = field_semantics or getattr(
- getattr(self, "_thread_local", None), "field_semantics", None
- )
-
- # Identify error items using KEYWORD detection (preservation guarantee)
- # This ensures ALL error items are kept, regardless of frequency
- error_indices = set(_detect_error_items_for_preservation(items))
-
- # Identify structural outlier indices using STATISTICAL detection
- # (items with rare fields or rare status values)
- outlier_indices = set(_detect_structural_outliers(items))
-
- # Identify numeric anomalies (MUST keep ALL of them)
- anomaly_indices = set()
- if analysis and analysis.field_stats:
- for field_name, stats in analysis.field_stats.items():
- if stats.field_type == "numeric" and stats.mean_val is not None and stats.variance:
- std = stats.variance**0.5
- if std > 0:
- threshold = self.config.variance_threshold * std
- for i, item in enumerate(items):
- val = item.get(field_name)
- if isinstance(val, int | float):
- if abs(val - stats.mean_val) > threshold:
- anomaly_indices.add(i)
-
- # === TOIN Evolution: Identify items with important values (learned) ===
- # Uses learned field semantics for zero-latency signal detection
- learned_important_indices: set[int] = set()
- if effective_field_semantics:
- learned_important_indices = set(
- _detect_items_by_learned_semantics(items, effective_field_semantics)
- )
-
- # Start with all critical items (these are non-negotiable)
- # Error items are ALWAYS preserved (quality guarantee)
- prioritized = error_indices | outlier_indices | anomaly_indices | learned_important_indices
-
- # HIGH FIX: Log warning if critical items alone exceed the limit
- # This helps diagnose why compression may be less effective than expected
- critical_count = len(prioritized)
- if critical_count > effective_max:
- logger.warning(
- "Critical items (%d) exceed max_items (%d): errors=%d outliers=%d anomalies=%d learned=%d. "
- "Quality guarantee takes precedence - keeping all critical items.",
- critical_count,
- effective_max,
- len(error_indices),
- len(outlier_indices),
- len(anomaly_indices),
- len(learned_important_indices),
- )
-
- # Add first/last items if we have room
- remaining_slots = effective_max - len(prioritized)
- if remaining_slots > 0:
- # First 3 items
- for i in range(min(3, n)):
- if i not in prioritized and remaining_slots > 0:
- prioritized.add(i)
- remaining_slots -= 1
- # Last 2 items
- for i in range(max(0, n - 2), n):
- if i not in prioritized and remaining_slots > 0:
- prioritized.add(i)
- remaining_slots -= 1
-
- # Fill remaining slots with other important indices (by index order)
- if remaining_slots > 0:
- other_indices = sorted(keep_indices - prioritized)
- for i in other_indices:
- if remaining_slots <= 0:
- break
- prioritized.add(i)
- remaining_slots -= 1
-
- return prioritized
-
- def should_apply(
- self,
- messages: list[dict[str, Any]],
- tokenizer: Tokenizer,
- **kwargs: Any,
- ) -> bool:
- """Check if any tool messages would benefit from smart crushing."""
- if not self.config.enabled:
- return False
-
- for msg in messages:
- # OpenAI style: role="tool"
- if msg.get("role") == "tool":
- content = msg.get("content", "")
- if isinstance(content, str):
- tokens = tokenizer.count_text(content)
- if tokens > self.config.min_tokens_to_crush:
- # Check if it's JSON with arrays
- parsed, success = safe_json_loads(content)
- if success and self._has_crushable_arrays(parsed):
- return True
-
- # Anthropic style: role="user" with tool_result content blocks
- content = msg.get("content")
- if isinstance(content, list):
- for block in content:
- if isinstance(block, dict) and block.get("type") == "tool_result":
- tool_content = block.get("content", "")
- if isinstance(tool_content, str):
- tokens = tokenizer.count_text(tool_content)
- if tokens > self.config.min_tokens_to_crush:
- parsed, success = safe_json_loads(tool_content)
- if success and self._has_crushable_arrays(parsed):
- return True
-
- return False
-
- def _has_crushable_arrays(self, data: Any, depth: int = 0) -> bool:
- """Check if data contains arrays large enough to crush.
-
- Accepts arrays of ANY homogeneous type (dicts, strings, numbers, etc.)
- as well as mixed-type arrays. Bool-only arrays are excluded (not useful
- to compress).
- """
- if depth > 5:
- return False
-
- if isinstance(data, list):
- if len(data) >= self.config.min_items_to_analyze:
- arr_type = _classify_array(data)
- if arr_type not in (ArrayType.EMPTY, ArrayType.BOOL_ARRAY):
- return True
- for item in data[:10]: # Check first few items
- if self._has_crushable_arrays(item, depth + 1):
- return True
-
- elif isinstance(data, dict):
- # Large objects with many keys are themselves crushable
- if len(data) >= self.config.min_items_to_analyze:
- return True
- for value in data.values():
- if self._has_crushable_arrays(value, depth + 1):
- return True
-
- return False
-
- def apply(
- self,
- messages: list[dict[str, Any]],
- tokenizer: Tokenizer,
- **kwargs: Any,
- ) -> TransformResult:
- """Apply smart crushing to messages."""
- tokens_before = tokenizer.count_messages(messages)
- result_messages = deep_copy_messages(messages)
- transforms_applied: list[str] = []
- markers_inserted: list[str] = []
- warnings: list[str] = []
-
- # Extract query context from recent user messages for relevance scoring
- query_context = self._extract_context_from_messages(result_messages)
-
- crushed_count = 0
- frozen_message_count = kwargs.get("frozen_message_count", 0)
-
- for msg_idx, msg in enumerate(result_messages):
- # Skip frozen messages (in provider's prefix cache)
- if msg_idx < frozen_message_count:
- continue
-
- # OpenAI style
- if msg.get("role") == "tool":
- content = msg.get("content", "")
- if not isinstance(content, str):
- continue
-
- tokens = tokenizer.count_text(content)
- if tokens <= self.config.min_tokens_to_crush:
- continue
-
- crushed, was_modified, analysis_info = self._smart_crush_content(
- content, query_context
- )
-
- if was_modified:
- original_hash = compute_short_hash(content)
- marker = create_tool_digest_marker(original_hash)
- msg["content"] = crushed + "\n" + marker
- crushed_count += 1
- markers_inserted.append(marker)
- if analysis_info:
- transforms_applied.append(f"smart:{analysis_info}")
-
- # Anthropic style
- content = msg.get("content")
- if isinstance(content, list):
- for i, block in enumerate(content):
- if not isinstance(block, dict):
- continue
- if block.get("type") != "tool_result":
- continue
-
- tool_content = block.get("content", "")
- if not isinstance(tool_content, str):
- continue
-
- tokens = tokenizer.count_text(tool_content)
- if tokens <= self.config.min_tokens_to_crush:
- continue
-
- crushed, was_modified, analysis_info = self._smart_crush_content(
- tool_content, query_context
- )
-
- if was_modified:
- original_hash = compute_short_hash(tool_content)
- marker = create_tool_digest_marker(original_hash)
- content[i]["content"] = crushed + "\n" + marker
- crushed_count += 1
- markers_inserted.append(marker)
- if analysis_info:
- transforms_applied.append(f"smart:{analysis_info}")
-
- if crushed_count > 0:
- transforms_applied.insert(0, f"smart_crush:{crushed_count}")
-
- tokens_after = tokenizer.count_messages(result_messages)
-
- return TransformResult(
- messages=result_messages,
- tokens_before=tokens_before,
- tokens_after=tokens_after,
- transforms_applied=transforms_applied,
- markers_inserted=markers_inserted,
- warnings=warnings,
- )
-
- def _extract_context_from_messages(self, messages: list[dict[str, Any]]) -> str:
- """Extract query context from recent messages for relevance scoring.
-
- Builds a context string from:
- - Recent user messages (what the user is asking about)
- - Recent tool call arguments (what data was requested)
-
- This context is used by RelevanceScorer to determine which items
- to preserve during crushing.
-
- Args:
- messages: Full message list.
-
- Returns:
- Context string for relevance scoring.
- """
- context_parts: list[str] = []
-
- # Look at last 5 user messages (most relevant to recent tool calls)
- user_message_count = 0
- for msg in reversed(messages):
- if msg.get("role") == "user":
- content = msg.get("content")
- if isinstance(content, str):
- context_parts.append(content)
- elif isinstance(content, list):
- # Anthropic style - extract from text blocks
- for block in content:
- if isinstance(block, dict) and block.get("type") == "text":
- text = block.get("text", "")
- if text:
- context_parts.append(text)
-
- user_message_count += 1
- if user_message_count >= 5:
- break
-
- # Also check assistant tool_calls for function arguments
- if msg.get("role") == "assistant" and msg.get("tool_calls"):
- for tc in msg.get("tool_calls", []):
- if isinstance(tc, dict):
- func = tc.get("function", {})
- args = func.get("arguments", "")
- if isinstance(args, str) and args:
- context_parts.append(args)
-
- return " ".join(context_parts)
-
- def _smart_crush_content(
- self,
- content: str,
- query_context: str = "",
- tool_name: str | None = None,
- bias: float = 1.0,
- ) -> tuple[str, bool, str]:
- """
- Apply smart crushing to content.
-
- Handles both JSON (existing SmartCrusher logic) and plain text content
- (search results, logs, generic text) using specialized compressors.
-
- Args:
- content: Content to crush (JSON or plain text).
- query_context: Context string from user messages for relevance scoring.
- tool_name: Name of the tool that produced this output.
- bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer).
-
- Returns:
- Tuple of (crushed_content, was_modified, analysis_info).
- """
- parsed, success = safe_json_loads(content)
- if not success:
- # Not JSON - pass through unchanged
- # Text compression utilities (SearchCompressor, LogCompressor, TextCompressor)
- # are available as standalone tools for applications to use explicitly
- return content, False, ""
-
- # Recursively process and crush arrays
- crushed, info, ccr_markers = self._process_value(
- parsed, query_context=query_context, tool_name=tool_name, bias=bias
- )
-
- result = safe_json_dumps(crushed, indent=None)
- was_modified = result != content.strip()
-
- # CCR: Inject retrieval markers if compression happened and CCR is enabled
- if was_modified and ccr_markers and self._ccr_config.inject_retrieval_marker:
- for marker_data in ccr_markers:
- if len(marker_data) == 4:
- ccr_hash, original_count, compressed_count, dropped_summary = marker_data
- else:
- ccr_hash, original_count, compressed_count = marker_data
- dropped_summary = ""
- summary_str = f" Omitted: {dropped_summary}." if dropped_summary else ""
- # Escape { } in summary to prevent .format() errors
- safe_summary = summary_str.replace("{", "{{").replace("}", "}}")
- ttl_seconds = getattr(self._ccr_config, "store_ttl_seconds", 300)
- marker = self._ccr_config.marker_template.format(
- original_count=original_count,
- compressed_count=compressed_count,
- hash=ccr_hash,
- summary=safe_summary,
- ttl_minutes=max(1, ttl_seconds // 60),
- )
- result += marker
-
- return result, was_modified, info
-
- # Maximum recursion depth for nested JSON processing.
- # Prevents RecursionError on adversarial/deeply-nested input.
- _MAX_PROCESS_DEPTH = 50
-
- def _process_value(
- self,
- value: Any,
- depth: int = 0,
- query_context: str = "",
- tool_name: str | None = None,
- bias: float = 1.0,
- ) -> tuple[Any, str, list[tuple[str, int, int]]]:
- """Recursively process a value, crushing arrays where appropriate.
-
- Returns:
- Tuple of (processed_value, info_string, ccr_markers).
- ccr_markers is a list of (hash, original_count, compressed_count, summary) tuples.
- """
- # Guard against deeply nested JSON causing RecursionError
- if depth >= self._MAX_PROCESS_DEPTH:
- return value, "", []
-
- info_parts = []
- ccr_markers: list[tuple] = []
-
- if isinstance(value, list):
- if len(value) >= self.config.min_items_to_analyze:
- arr_type = _classify_array(value)
-
- if arr_type == ArrayType.DICT_ARRAY:
- # Existing path — dict arrays (battle-tested, unchanged)
- crushed, strategy, ccr_hash, dropped_summary = self._crush_array(
- value, query_context, tool_name, bias=bias
- )
- info_parts.append(f"{strategy}({len(value)}->{len(crushed)})")
- if ccr_hash:
- ccr_markers.append((ccr_hash, len(value), len(crushed), dropped_summary))
- return crushed, ",".join(info_parts), ccr_markers
-
- elif arr_type == ArrayType.STRING_ARRAY:
- crushed, strategy = self._crush_string_array(value, bias=bias)
- info_parts.append(f"{strategy}({len(value)}->{len(crushed)})")
- return crushed, ",".join(info_parts), ccr_markers
-
- elif arr_type == ArrayType.NUMBER_ARRAY:
- crushed, strategy = self._crush_number_array(value, bias=bias)
- if isinstance(crushed, list):
- info_parts.append(f"{strategy}({len(value)}->{len(crushed)})")
- else:
- info_parts.append(f"{strategy}({len(value)}->summary)")
- return crushed, ",".join(info_parts), ccr_markers
-
- elif arr_type == ArrayType.MIXED_ARRAY:
- crushed, strategy = self._crush_mixed_array(
- value, query_context, tool_name, bias=bias
- )
- info_parts.append(f"{strategy}({len(value)}->{len(crushed)})")
- return crushed, ",".join(info_parts), ccr_markers
-
- # NESTED_ARRAY, BOOL_ARRAY, EMPTY — fall through to recursive
-
- # Not crushable or below threshold — process items recursively
- processed = []
- for item in value:
- p_item, p_info, p_markers = self._process_value(
- item, depth + 1, query_context, tool_name, bias=bias
- )
- processed.append(p_item)
- if p_info:
- info_parts.append(p_info)
- ccr_markers.extend(p_markers)
- return processed, ",".join(info_parts), ccr_markers
-
- elif isinstance(value, dict):
- # First: recurse into values to compress nested arrays
- processed_dict: dict[str, Any] = {}
- for k, v in value.items():
- p_val, p_info, p_markers = self._process_value(
- v, depth + 1, query_context, tool_name, bias=bias
- )
- processed_dict[k] = p_val
- if p_info:
- info_parts.append(p_info)
- ccr_markers.extend(p_markers)
-
- # Second: if the object itself has many keys, compress at key level
- if len(processed_dict) >= self.config.min_items_to_analyze:
- crushed_dict, strategy = self._crush_object(processed_dict, bias=bias)
- if strategy != "object:passthrough":
- info_parts.append(strategy)
- return crushed_dict, ",".join(info_parts), ccr_markers
-
- return processed_dict, ",".join(info_parts), ccr_markers
-
- else:
- return value, "", []
-
- def _crush_array(
- self,
- items: list[dict],
- query_context: str = "",
- tool_name: str | None = None,
- bias: float = 1.0,
- ) -> tuple[list, str, str | None, str]:
- """Crush an array using statistical analysis and relevance scoring.
-
- IMPORTANT: If crushability analysis determines it's not safe to crush
- (high variability + no importance signal), returns original array unchanged.
-
- TOIN-aware: Consults the Tool Output Intelligence Network for cross-user
- learned patterns. High retrieval rate across all users → compress less.
-
- Feedback-aware: Uses learned patterns to adjust compression aggressiveness.
- High retrieval rate for a tool → compress less aggressively.
-
- Args:
- items: List of dict items to compress.
- query_context: Context string from user messages for relevance scoring.
- tool_name: Name of the tool that produced this output.
- bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer).
-
- Returns:
- Tuple of (crushed_items, strategy_info, ccr_hash, dropped_summary).
- ccr_hash is the hash for retrieval if CCR is enabled, None otherwise.
- dropped_summary is a categorical summary of what was dropped.
- """
- # BOUNDARY CHECK: Use adaptive sizing instead of hardcoded limit
- # compute_optimal_k handles trivial cases (n <= 8 → keep all)
- from .adaptive_sizer import compute_optimal_k
-
- item_strings = [json.dumps(item, default=str) for item in items]
- adaptive_k = compute_optimal_k(
- item_strings,
- bias=bias,
- min_k=3,
- max_k=self.config.max_items_after_crush if self.config.max_items_after_crush else None,
- )
-
- if len(items) <= adaptive_k:
- # All items kept (high diversity or small array).
- # Instead of passing through unchanged, try to compress the TEXT
- # WITHIN each item — reduce token count without losing any item.
- compressed_items = _compress_text_within_items(items, query_context)
- if compressed_items is not items:
- return compressed_items, "compress_within:diversity", None, ""
- return items, "none:adaptive_at_limit", None, ""
-
- # Get feedback hints if enabled
- # THREAD-SAFETY: Use a local effective_max_items instead of mutating shared config
- effective_max_items = adaptive_k
- hints_applied = False
- toin_hint_applied = False
-
- # Create ToolSignature for TOIN lookup
- tool_signature = ToolSignature.from_items(items)
-
- # TOIN: Get cross-user learned recommendations
- toin = self._get_toin()
- toin_hint = toin.get_recommendation(tool_signature, query_context)
-
- # Log TOIN hint details
- logger.debug(
- "TOIN hint: source=%s, confidence=%.2f, skip=%s, max_items=%d",
- toin_hint.source,
- toin_hint.confidence,
- toin_hint.skip_compression,
- toin_hint.max_items,
- )
-
- if toin_hint.skip_compression:
- return items, f"skip:toin({toin_hint.reason})", None, ""
-
- # Apply TOIN recommendations if from network or local learning
- toin_preserve_fields: list[str] = []
- toin_recommended_strategy: str | None = None
- toin_compression_level: str | None = None
- # LOW FIX #21: Use configurable threshold instead of hardcoded 0.5
- if (
- toin_hint.source in ("network", "local")
- and toin_hint.confidence >= self.config.toin_confidence_threshold
- ):
- # TOIN recommendations take precedence over local feedback
- effective_max_items = toin_hint.max_items
- toin_preserve_fields = toin_hint.preserve_fields # Fields to never remove
- toin_hint_applied = True
- # Store strategy and compression level for later use
- if toin_hint.recommended_strategy != "default":
- toin_recommended_strategy = toin_hint.recommended_strategy
- if toin_hint.compression_level != "moderate":
- toin_compression_level = toin_hint.compression_level
- # Log that TOIN hint was applied
- logger.debug(
- "TOIN hint applied: max_items=%d, strategy=%s, compression_level=%s",
- effective_max_items,
- toin_recommended_strategy or "default",
- toin_compression_level or "moderate",
- )
- elif toin_hint.source in ("network", "local"):
- # Hint available but confidence too low
- logger.debug(
- "TOIN hint not applied: confidence %.2f < threshold %.2f",
- toin_hint.confidence,
- self.config.toin_confidence_threshold,
- )
-
- # === TOIN Evolution: Extract field semantics for signal detection ===
- # Store in thread-local storage for use in _prioritize_indices.
- # This enables learned signal detection without changing all method signatures
- # while remaining thread-safe (no cross-thread contamination).
- if not hasattr(self, "_thread_local"):
- self._thread_local = threading.local()
- self._thread_local.field_semantics = (
- toin_hint.field_semantics if toin_hint.field_semantics else None
- )
-
- # Local feedback hints (if TOIN didn't apply)
- if not toin_hint_applied and self.config.use_feedback_hints and tool_name:
- feedback = self._get_feedback()
- hints = feedback.get_compression_hints(tool_name)
-
- # Check if hints recommend skipping compression
- if hints.skip_compression:
- return items, f"skip:feedback({hints.reason})", None, ""
-
- # Adjust max_items based on feedback
- if hints.suggested_items is not None:
- effective_max_items = hints.suggested_items
- hints_applied = True
-
- # Use preserve_fields from local feedback (hash them for TOIN compatibility)
- # Note: CompressionFeedback stores actual field names, but _plan methods
- # expect SHA256[:8] hashes for privacy-preserving comparison
- if hints.preserve_fields:
- toin_preserve_fields = [_hash_field_name(field) for field in hints.preserve_fields]
-
- # Use recommended_strategy from local feedback if not already set by TOIN
- if hints.recommended_strategy and not toin_recommended_strategy:
- toin_recommended_strategy = hints.recommended_strategy
-
- try:
- # Analyze the array (includes crushability check)
- analysis = self.analyzer.analyze_array(items)
-
- # CRITICAL: If not crushable, return original array unchanged
- if analysis.recommended_strategy == CompressionStrategy.SKIP:
- reason = ""
- if analysis.crushability:
- reason = f"skip:{analysis.crushability.reason}"
- return items, reason, None, ""
-
- # Apply TOIN strategy recommendation if available
- # TOIN learns which strategies work best from cross-user patterns
- if toin_recommended_strategy:
- try:
- toin_strategy = CompressionStrategy(toin_recommended_strategy)
- # Only override if TOIN suggests a valid non-SKIP strategy
- if toin_strategy != CompressionStrategy.SKIP:
- analysis.recommended_strategy = toin_strategy
- except ValueError:
- pass # Invalid strategy name, keep analyzer's choice
-
- # Apply TOIN compression level to adjust effective_max_items
- if toin_compression_level:
- if toin_compression_level == "none":
- # Don't compress - return original
- return items, "skip:toin_level_none", None, ""
- elif toin_compression_level == "conservative":
- # Be conservative - keep more items
- effective_max_items = max(effective_max_items, min(50, len(items) // 2))
- elif toin_compression_level == "aggressive":
- # Be aggressive - keep fewer items
- effective_max_items = min(effective_max_items, 15)
-
- # Create compression plan with relevance scoring
- # Pass TOIN preserve_fields so items with those fields get priority
- # Pass effective_max_items for thread-safe compression
- # Pass item_strings to avoid redundant json.dumps across plan methods
- plan = self._create_plan(
- analysis,
- items,
- query_context,
- preserve_fields=toin_preserve_fields or None,
- effective_max_items=effective_max_items,
- item_strings=item_strings,
- )
-
- # Execute compression
- result = self._execute_plan(plan, items, analysis)
-
- # CCR: Store original content for retrieval if enabled
- ccr_hash = None
- if (
- self._ccr_config.enabled
- and len(items) >= self._ccr_config.min_items_to_cache
- and len(result) < len(items) # Only cache if compression actually happened
- ):
- store = self._get_compression_store()
- # Reuse cached item_strings to avoid re-serializing
- original_json = "[" + ", ".join(item_strings) + "]"
- compressed_json = json.dumps(result, default=str)
-
- ccr_hash = store.store(
- original=original_json,
- compressed=compressed_json,
- original_item_count=len(items),
- compressed_item_count=len(result),
- tool_name=tool_name,
- query_context=query_context,
- # CRITICAL: Pass the tool_signature_hash so retrieval events
- # can be correlated with compression events in TOIN
- tool_signature_hash=tool_signature.structure_hash,
- compression_strategy=analysis.recommended_strategy.value,
- )
-
- # Record compression event for feedback loop
- if self.config.use_feedback_hints and tool_name:
- feedback = self._get_feedback()
- feedback.record_compression(
- tool_name=tool_name,
- original_count=len(items),
- compressed_count=len(result),
- strategy=analysis.recommended_strategy.value,
- tool_signature_hash=tool_signature.structure_hash,
- )
-
- # Record telemetry for data flywheel
- self._record_telemetry(
- items=items,
- result=result,
- analysis=analysis,
- plan=plan,
- tool_name=tool_name,
- )
-
- # TOIN: Record compression event for cross-user learning
- try:
- # Calculate token counts (approximate) - reuse cached item_strings
- original_tokens = sum(len(s) for s in item_strings) // 4
- compressed_tokens = len(json.dumps(result, default=str)) // 4
-
- toin.record_compression(
- tool_signature=tool_signature,
- original_count=len(items),
- compressed_count=len(result),
- original_tokens=original_tokens,
- compressed_tokens=compressed_tokens,
- strategy=analysis.recommended_strategy.value,
- query_context=query_context,
- items=items, # Pass items for field-level semantic learning
- )
- except Exception:
- # TOIN should never break compression
- pass
-
- strategy_info = analysis.recommended_strategy.value
- if toin_hint_applied:
- toin_parts = [f"items={toin_hint.max_items}", f"conf={toin_hint.confidence:.2f}"]
- if toin_recommended_strategy:
- toin_parts.append(f"strategy={toin_recommended_strategy}")
- if toin_compression_level and toin_compression_level != "moderate":
- toin_parts.append(f"level={toin_compression_level}")
- strategy_info += f"(toin:{','.join(toin_parts)})"
- elif hints_applied:
- strategy_info += f"(feedback:{effective_max_items})"
-
- # Generate categorical summary of dropped items (use indices, not identity)
- from .compression_summary import summarize_dropped_items
-
- dropped_summary = summarize_dropped_items(
- items,
- result,
- kept_indices=set(plan.keep_indices),
- )
-
- # Clean up temporary instance variable
- if hasattr(self, "_thread_local"):
- self._thread_local.field_semantics = None
- return result, strategy_info, ccr_hash, dropped_summary
-
- except Exception:
- # Clean up temporary instance variable
- if hasattr(self, "_thread_local"):
- self._thread_local.field_semantics = None
- # Re-raise any exceptions (removed finally block since we no longer mutate config)
- raise
-
- # =================================================================
- # Universal JSON type handlers (string, number, mixed arrays)
- # =================================================================
-
- def _compute_k_split(
- self,
- items: list,
- bias: float = 1.0,
- item_strings: list[str] | None = None,
- ) -> tuple[int, int, int, int]:
- """Compute adaptive K split into first/last/importance slots.
-
- Uses the existing Kneedle-based adaptive_sizer for K_total, then
- splits according to configurable first_fraction / last_fraction.
-
- Args:
- items: List of items (used as fallback for serialization).
- bias: Compression bias multiplier.
- item_strings: Pre-computed JSON serializations to avoid redundant json.dumps.
-
- Returns:
- (k_total, k_first, k_last, k_importance)
- """
- from .adaptive_sizer import compute_optimal_k
-
- if item_strings is None:
- item_strings = [json.dumps(item, default=str) for item in items]
- k_total = compute_optimal_k(
- item_strings,
- bias=bias,
- min_k=3,
- max_k=self.config.max_items_after_crush or None,
- )
- # BUG #4 FIX: clamp k_first and k_last so their sum never exceeds
- # k_total. Without the clamp, k_total=1 produces k_first=k_last=1
- # (both `max(1, round(0))`), violating max_items_after_crush.
- # No-op for k_total >= 2 (the common path).
- k_first = max(1, round(k_total * self.config.first_fraction))
- k_first = min(k_first, k_total)
- k_last = max(1, round(k_total * self.config.last_fraction))
- k_last = min(k_last, max(0, k_total - k_first))
- k_importance = max(0, k_total - k_first - k_last)
- return k_total, k_first, k_last, k_importance
-
- def _crush_string_array(
- self,
- items: list[str],
- bias: float = 1.0,
- ) -> tuple[list[str], str]:
- """Crush an array of strings using dedup + adaptive sampling.
-
- Strategy:
- 1. Compute adaptive K via Kneedle algorithm
- 2. Always keep: error-containing strings, first K, last K
- 3. Deduplicate exact matches
- 4. Fill remaining budget with diverse samples (stride-based)
-
- Returns:
- (crushed_items, strategy_string)
- """
- n = len(items)
- if n <= 8:
- return items, "string:passthrough"
-
- k_total, k_first, k_last, k_importance = self._compute_k_split(items, bias)
-
- # Mandatory: error-containing strings (never dropped)
- error_indices: set[int] = set()
- for i, s in enumerate(items):
- s_lower = s.lower()
- for keyword in _ERROR_KEYWORDS_FOR_PRESERVATION:
- if keyword in s_lower:
- error_indices.add(i)
- break
-
- # Mandatory: strings with abnormal length (anomalies)
- lengths = [len(s) for s in items]
- if len(lengths) > 1:
- mean_len = statistics.mean(lengths)
- std_len = statistics.stdev(lengths)
- anomaly_indices = {
- i
- for i, length in enumerate(lengths)
- if std_len > 0 and abs(length - mean_len) > self.config.variance_threshold * std_len
- }
- else:
- anomaly_indices = set[int]()
-
- # Boundary: first K, last K
- first_indices = set(range(min(k_first, n)))
- last_indices = set(range(max(0, n - k_last), n))
-
- # Combine mandatory + boundary
- keep_indices = error_indices | anomaly_indices | first_indices | last_indices
-
- # Dedup: among remaining candidates, skip exact duplicates
- seen_strings: set[str] = set()
- dedup_count = 0
- for i in sorted(keep_indices):
- seen_strings.add(items[i])
-
- # Fill remaining budget with diverse stride-based samples
- remaining_budget = max(0, k_total - len(keep_indices))
- if remaining_budget > 0:
- stride = max(1, (n - 1) // (remaining_budget + 1))
- for i in range(0, n, stride):
- if len(keep_indices) >= k_total + len(error_indices) + len(anomaly_indices):
- break
- if i not in keep_indices:
- if items[i] not in seen_strings:
- keep_indices.add(i)
- seen_strings.add(items[i])
- else:
- dedup_count += 1
-
- # Build output in original order
- result = [items[i] for i in sorted(keep_indices)]
-
- strategy = f"string:adaptive({n}->{len(result)}"
- if dedup_count:
- strategy += f",dedup={dedup_count}"
- if error_indices:
- strategy += f",errors={len(error_indices)}"
- strategy += ")"
-
- return result, strategy
-
- def _crush_number_array(
- self,
- items: list[int | float],
- bias: float = 1.0,
- ) -> tuple[list, str]:
- """Crush an array of numbers using statistical summary + outlier preservation.
-
- Strategy:
- 1. Compute descriptive statistics (min, max, mean, median, stddev, percentiles)
- 2. Detect outliers (> variance_threshold σ from mean)
- 3. Detect change points (sudden shifts in running mean)
- 4. Keep: first K, last K, all outliers, change points
- 5. Return kept values with a prepended stats summary string
-
- Returns:
- (crushed_items, strategy_string) where crushed_items is a list
- starting with a summary string followed by representative values.
- """
- n = len(items)
- if n <= 8:
- return items, "number:passthrough"
-
- # Filter out non-finite values for statistics
- finite = [x for x in items if isinstance(x, int | float) and math.isfinite(x)]
- if not finite:
- return items, "number:no_finite"
-
- k_total, k_first, k_last, k_importance = self._compute_k_split(items, bias)
-
- # Statistics
- mean_val = statistics.mean(finite)
- median_val = statistics.median(finite)
- std_val = statistics.stdev(finite) if len(finite) > 1 else 0.0
- sorted_finite = sorted(finite)
- # BUG #1 FIX: replace integer-division indexing (off-by-one
- # for len < 8) with proper linear-interpolation percentile.
- # `_percentile_linear` matches numpy's "linear" method exactly
- # — index = q * (n - 1), interpolate between floor and ceil.
- p25 = _percentile_linear(sorted_finite, 0.25)
- p75 = _percentile_linear(sorted_finite, 0.75)
-
- # Outliers (> variance_threshold σ from mean)
- outlier_indices: set[int] = set()
- if std_val > 0:
- for i, val in enumerate(items):
- if isinstance(val, int | float) and math.isfinite(val):
- if abs(val - mean_val) > self.config.variance_threshold * std_val:
- outlier_indices.add(i)
-
- # Change points (detect sudden shifts using running difference)
- change_indices: set[int] = set()
- if self.config.preserve_change_points and n > 10:
- window = 5
- for i in range(window, n - window):
- left = [
- items[j]
- for j in range(i - window, i)
- if isinstance(items[j], int | float) and math.isfinite(items[j])
- ]
- right = [
- items[j]
- for j in range(i, i + window)
- if isinstance(items[j], int | float) and math.isfinite(items[j])
- ]
- if left and right:
- left_mean = statistics.mean(left)
- right_mean = statistics.mean(right)
- if (
- std_val > 0
- and abs(right_mean - left_mean) > self.config.variance_threshold * std_val
- ):
- change_indices.add(i)
-
- # Boundary: first K, last K
- first_indices = set(range(min(k_first, n)))
- last_indices = set(range(max(0, n - k_last), n))
-
- # Combine all
- keep_indices = outlier_indices | change_indices | first_indices | last_indices
-
- # Fill remaining budget with stride-based samples
- remaining_budget = max(0, k_total - len(keep_indices))
- if remaining_budget > 0:
- stride = max(1, (n - 1) // (remaining_budget + 1))
- for i in range(0, n, stride):
- if len(keep_indices) >= k_total + len(outlier_indices):
- break
- if i not in keep_indices:
- keep_indices.add(i)
-
- # Build output: kept values only (schema-preserving — no generated text)
- kept_values = [items[i] for i in sorted(keep_indices)]
-
- # Encode statistics into the strategy string (not the array itself)
- strategy = (
- f"number:adaptive({n}->{len(kept_values)}"
- f",min={min(finite)},max={max(finite)}"
- f",mean={mean_val:.4g},median={median_val:.4g}"
- f",stddev={std_val:.4g},p25={p25:.4g},p75={p75:.4g}"
- )
- if outlier_indices:
- strategy += f",outliers={len(outlier_indices)}"
- if change_indices:
- strategy += f",change_points={len(change_indices)}"
- strategy += ")"
-
- return kept_values, strategy
-
- def _crush_mixed_array(
- self,
- items: list,
- query_context: str = "",
- tool_name: str | None = None,
- bias: float = 1.0,
- ) -> tuple[list, str]:
- """Crush a mixed-type array by grouping items by type and compressing each group.
-
- Strategy:
- 1. Group items by type (dict, str, number, list, None, bool)
- 2. For each group with >= min_items_to_analyze items: compress with appropriate handler
- 3. For small groups: keep all items
- 4. Reassemble in original order
-
- Returns:
- (crushed_items, strategy_string)
- """
- n = len(items)
- if n <= 8:
- return items, "mixed:passthrough"
-
- # Group items by type, tracking original indices
- groups: dict[str, list[tuple[int, Any]]] = {}
- for i, item in enumerate(items):
- if isinstance(item, dict):
- key = "dict"
- elif isinstance(item, str):
- key = "str"
- elif isinstance(item, bool):
- key = "bool"
- elif isinstance(item, int | float):
- key = "number"
- elif isinstance(item, list):
- key = "list"
- elif item is None:
- key = "none"
- else:
- key = "other"
- groups.setdefault(key, []).append((i, item))
-
- # Compress each group independently
- keep_indices: set[int] = set()
- strategy_parts: list[str] = []
-
- for type_key, group_items in groups.items():
- indices = [idx for idx, _ in group_items]
- values = [val for _, val in group_items]
-
- if len(values) < self.config.min_items_to_analyze:
- # Small group — keep all
- keep_indices.update(indices)
- continue
-
- if type_key == "dict":
- # Use existing dict array crusher
- crushed, strategy, _, _ = self._crush_array(
- values, query_context, tool_name, bias=bias
- )
- crushed_set = {json.dumps(c, sort_keys=True, default=str) for c in crushed}
- for idx, val in group_items:
- if json.dumps(val, sort_keys=True, default=str) in crushed_set:
- keep_indices.add(idx)
- strategy_parts.append(f"dict:{len(values)}->{len(crushed)}")
-
- elif type_key == "str":
- crushed, strategy = self._crush_string_array(values, bias=bias)
- crushed_set = set(crushed)
- for idx, val in group_items:
- if val in crushed_set:
- keep_indices.add(idx)
- strategy_parts.append(f"str:{len(values)}->{len(crushed)}")
-
- elif type_key == "number":
- # For numbers in mixed arrays, just do adaptive sampling (no summary prefix)
- k_total, k_first, k_last, _ = self._compute_k_split(values, bias)
- first_idx = set(indices[:k_first])
- last_idx = set(indices[-k_last:])
- keep_indices.update(first_idx | last_idx)
- # Outliers
- finite = [v for v in values if isinstance(v, int | float) and math.isfinite(v)]
- if len(finite) > 1:
- mean_v = statistics.mean(finite)
- std_v = statistics.stdev(finite)
- if std_v > 0:
- for idx, val in group_items:
- if isinstance(val, int | float) and math.isfinite(val):
- if abs(val - mean_v) > self.config.variance_threshold * std_v:
- keep_indices.add(idx)
- strategy_parts.append(f"num:{len(values)}")
+"""Smart JSON array crusher — Rust-backed via PyO3.
+
+The Python implementation has been retired (Stage 3c.1b, 2026-04-27).
+All array compression now goes through `headroom._core.SmartCrusher`
+(built from `crates/headroom-py`). Byte-equality of the two
+implementations was verified against 17 recorded fixtures
+(`tests/parity/fixtures/smart_crusher/`) before the Python source was
+removed; the Rust crate has its own coverage in `crates/headroom-core/`
+(388 unit tests + property tests).
+
+This module retains the public surface — `SmartCrusherConfig`,
+`CrushResult`, `SmartCrusher`, `smart_crush_tool_output` — so existing
+call sites keep working unchanged. The dataclasses are still pure
+Python because callers use `asdict()`, `__dict__`, and dataclass
+matching on them. Only the `SmartCrusher` class delegates to Rust.
+
+The `headroom._core` extension is a hard import: there is no Python
+fallback. Build it locally with `scripts/build_rust_extension.sh`
+(wraps `maturin develop`) or install a prebuilt wheel.
+
+Stage 3c.1 deliberately keeps the optional subsystems (TOIN,
+feedback, CCR marker injection, telemetry) disabled in the Rust port.
+The shim accepts `relevance_config`, `scorer`, and `ccr_config`
+constructor args for source compatibility but does not wire them
+through — they re-attach in Stage 3c.2 when those subsystems land in
+Rust. CCR marker injection in `_smart_crush_content` is a Stage 3c.2
+follow-up; today the Rust port never emits CCR markers, so the
+disabled-path behavior is byte-equal.
+"""
- else:
- # list, bool, none, other — keep all
- keep_indices.update(indices)
+from __future__ import annotations
- # Reassemble in original order
- result = [items[i] for i in sorted(keep_indices)]
+import logging
+from dataclasses import dataclass
+from typing import Any
- strategy = f"mixed:adaptive({n}->{len(result)},{','.join(strategy_parts)})"
- return result, strategy
+from ..config import CCRConfig, TransformResult
+from ..tokenizer import Tokenizer
+from ..utils import compute_short_hash, create_tool_digest_marker, deep_copy_messages
+from .base import Transform
- def _crush_object(
- self,
- obj: dict[str, Any],
- bias: float = 1.0,
- ) -> tuple[dict[str, Any], str]:
- """Crush a large JSON object by selecting the most informative keys.
+logger = logging.getLogger(__name__)
- Treats key-value pairs as items and applies adaptive K to select which
- keys to retain. Preserves schema — each kept key-value pair is exact
- from the original.
- Strategy:
- 1. Classify each value by size (tokens) and importance
- 2. Always keep: keys with small values (cheap), keys with error content
- 3. Compute adaptive K on key-value representations
- 4. Fill remaining budget with diverse keys (stride-based)
+# ─── Public dataclasses ───────────────────────────────────────────────────
- Returns:
- (compressed_object, strategy_string)
- """
- n = len(obj)
- if n <= 8:
- return obj, "object:passthrough"
- # Estimate tokens per key-value pair
- kv_tokens: list[tuple[str, int]] = []
- total_tokens = 0
- for key, val in obj.items():
- val_str = json.dumps(val, default=str)
- tokens = len(val_str) // 4 + len(key) // 4 + 2 # rough estimate
- kv_tokens.append((key, tokens))
- total_tokens += tokens
+@dataclass
+class CrushResult:
+ """Result from `SmartCrusher.crush()`.
- # If already small enough, passthrough
- if total_tokens < self.config.min_tokens_to_crush:
- return obj, "object:passthrough"
+ Used by `ContentRouter` when routing JSON arrays to `SmartCrusher`.
+ """
- # Compute adaptive K on key-value string representations
- keys = list(obj.keys())
- kv_strings = [f"{k}: {json.dumps(obj[k], default=str)}" for k in keys]
+ compressed: str
+ original: str
+ was_modified: bool
+ strategy: str = "passthrough"
- from .adaptive_sizer import compute_optimal_k
- k_total = compute_optimal_k(
- kv_strings,
- bias=bias,
- min_k=3,
- max_k=self.config.max_items_after_crush or None,
- )
+@dataclass
+class SmartCrusherConfig:
+ """Configuration for SmartCrusher.
- if k_total >= n:
- return obj, "object:passthrough"
+ SCHEMA-PRESERVING: output contains only items from the original
+ array. No wrappers, no generated text, no metadata keys.
- # Classify keys by importance
- keep_keys: set[str] = set()
+ Field names + defaults match the Rust `SmartCrusherConfig` byte-for-
+ byte; the shim copies these straight into the PyO3 constructor.
+ """
- # Always keep: keys with error-containing values
- for key, val in obj.items():
- val_str = json.dumps(val, default=str).lower()
- for keyword in _ERROR_KEYWORDS_FOR_PRESERVATION:
- if keyword in val_str:
- keep_keys.add(key)
- break
+ enabled: bool = True
+ min_items_to_analyze: int = 5
+ min_tokens_to_crush: int = 200
+ variance_threshold: float = 2.0
+ uniqueness_threshold: float = 0.1
+ similarity_threshold: float = 0.8
+ max_items_after_crush: int = 15
+ preserve_change_points: bool = True
+ factor_out_constants: bool = False
+ include_summaries: bool = False
+ use_feedback_hints: bool = True
+ toin_confidence_threshold: float = 0.5
+ dedup_identical_items: bool = True
+ first_fraction: float = 0.3
+ last_fraction: float = 0.15
- # Always keep: keys with small values (cheap to keep)
- small_threshold = 50 # chars
- for key, tokens in kv_tokens:
- if tokens <= small_threshold // 4:
- keep_keys.add(key)
- # Boundary: first K and last K keys
- k_first = max(1, round(k_total * self.config.first_fraction))
- k_last = max(1, round(k_total * self.config.last_fraction))
- for key in keys[:k_first]:
- keep_keys.add(key)
- for key in keys[-k_last:]:
- keep_keys.add(key)
+# ─── Rust-backed SmartCrusher ─────────────────────────────────────────────
- # Fill remaining budget with stride-based diverse sampling
- remaining = max(0, k_total - len(keep_keys))
- if remaining > 0:
- stride = max(1, (n - 1) // (remaining + 1))
- for i in range(0, n, stride):
- if len(keep_keys) >= k_total + len(
- [
- k
- for k in keep_keys
- if any(
- kw in json.dumps(obj[k], default=str).lower()
- for kw in _ERROR_KEYWORDS_FOR_PRESERVATION
- )
- ]
- ):
- break
- keep_keys.add(keys[i])
- # Build output preserving original key order
- result = {k: obj[k] for k in keys if k in keep_keys}
+class SmartCrusher(Transform):
+ """Rust-backed `SmartCrusher` (via PyO3 / `headroom._core`).
+
+ Same `__init__` and method shapes as the retired Python class —
+ drop-in replacement. The `crush()` and `_smart_crush_content()`
+ methods delegate every byte to Rust; `apply()` keeps the
+ Transform-protocol orchestration in Python (message walking,
+ digest-marker insertion, token counting) since that's mostly glue
+ around the per-message compression call.
+ """
- strategy = f"object:adaptive({n}->{len(result)} keys)"
- return result, strategy
+ name = "smart_crusher"
- def _create_plan(
+ def __init__(
self,
- analysis: ArrayAnalysis,
- items: list[dict],
- query_context: str = "",
- preserve_fields: list[str] | None = None,
- effective_max_items: int | None = None,
- item_strings: list[str] | None = None,
- ) -> CompressionPlan:
- """Create a detailed compression plan using relevance scoring.
-
- Args:
- analysis: The array analysis results.
- items: The items to compress.
- query_context: Context string from user messages for relevance scoring.
- preserve_fields: TOIN-learned fields that users commonly retrieve.
- Items with values in these fields get higher priority.
- item_strings: Pre-computed JSON serializations to avoid redundant json.dumps.
- effective_max_items: Thread-safe max items limit (defaults to config value).
- """
- # Use provided effective_max_items or fall back to config
- max_items = (
- effective_max_items
- if effective_max_items is not None
- else self.config.max_items_after_crush
+ config: SmartCrusherConfig | None = None,
+ relevance_config: Any = None,
+ scorer: Any = None,
+ ccr_config: CCRConfig | None = None,
+ ):
+ # Hard import — no Python fallback. If the wheel is missing the
+ # caller must build it (scripts/build_rust_extension.sh) or
+ # install a prebuilt one. Failing loudly is better than silent
+ # degradation; see feedback memory `feedback_no_silent_fallbacks.md`.
+ from headroom._core import (
+ SmartCrusher as _RustSmartCrusher,
)
-
- plan = CompressionPlan(
- strategy=analysis.recommended_strategy,
- constant_fields=analysis.constant_fields if self.config.factor_out_constants else {},
+ from headroom._core import (
+ SmartCrusherConfig as _RustSmartCrusherConfig,
)
- # Handle SKIP - keep all items (shouldn't normally reach here)
- if analysis.recommended_strategy == CompressionStrategy.SKIP:
- plan.keep_indices = list(range(len(items)))
- return plan
-
- if analysis.recommended_strategy == CompressionStrategy.TIME_SERIES:
- plan = self._plan_time_series(
- analysis,
- items,
- plan,
- query_context,
- preserve_fields,
- max_items,
- item_strings=item_strings,
- )
+ cfg = config or SmartCrusherConfig()
+ self.config = cfg
- elif analysis.recommended_strategy == CompressionStrategy.CLUSTER_SAMPLE:
- plan = self._plan_cluster_sample(
- analysis,
- items,
- plan,
- query_context,
- preserve_fields,
- max_items,
- item_strings=item_strings,
- )
+ # CCR config is preserved on `self` for callers that read it
+ # back (`headroom.proxy.server` does), but the Rust port doesn't
+ # exercise it: Stage 3c.1 keeps CCR marker injection disabled
+ # because the Rust port has no compression store. When Stage
+ # 3c.2 lands the CCR port, this wires through to Rust.
+ if ccr_config is None:
+ self._ccr_config = CCRConfig(enabled=True, inject_retrieval_marker=False)
+ else:
+ self._ccr_config = ccr_config
- elif analysis.recommended_strategy == CompressionStrategy.TOP_N:
- plan = self._plan_top_n(
- analysis,
- items,
- plan,
- query_context,
- preserve_fields,
- max_items,
- item_strings=item_strings,
+ # `relevance_config` and `scorer` are accepted for source
+ # compatibility but currently dropped — Stage 3c.1 ships with
+ # the Rust default `HybridScorer`. Custom scorers re-attach in
+ # Stage 3c.2 when the relevance crate gains a Python-bridged
+ # constructor surface.
+ if relevance_config is not None or scorer is not None:
+ logger.debug(
+ "SmartCrusher: relevance_config/scorer args are ignored in "
+ "Stage 3c.1 (Rust port uses default HybridScorer). They "
+ "will be wired through in Stage 3c.2."
)
- else: # SMART_SAMPLE or NONE
- plan = self._plan_smart_sample(
- analysis,
- items,
- plan,
- query_context,
- preserve_fields,
- max_items,
- item_strings=item_strings,
+ # Build the Rust crusher with every field from the Python
+ # config, plus the relevance_threshold default (0.3) — the
+ # Python dataclass doesn't carry that field; it lives on
+ # `RelevanceScorerConfig` instead.
+ self._rust = _RustSmartCrusher(
+ _RustSmartCrusherConfig(
+ enabled=cfg.enabled,
+ min_items_to_analyze=cfg.min_items_to_analyze,
+ min_tokens_to_crush=cfg.min_tokens_to_crush,
+ variance_threshold=cfg.variance_threshold,
+ uniqueness_threshold=cfg.uniqueness_threshold,
+ similarity_threshold=cfg.similarity_threshold,
+ max_items_after_crush=cfg.max_items_after_crush,
+ preserve_change_points=cfg.preserve_change_points,
+ factor_out_constants=cfg.factor_out_constants,
+ include_summaries=cfg.include_summaries,
+ use_feedback_hints=cfg.use_feedback_hints,
+ toin_confidence_threshold=cfg.toin_confidence_threshold,
+ dedup_identical_items=cfg.dedup_identical_items,
+ first_fraction=cfg.first_fraction,
+ last_fraction=cfg.last_fraction,
+ relevance_threshold=0.3,
)
-
- return plan
-
- def _plan_time_series(
- self,
- analysis: ArrayAnalysis,
- items: list[dict],
- plan: CompressionPlan,
- query_context: str = "",
- preserve_fields: list[str] | None = None,
- max_items: int | None = None,
- item_strings: list[str] | None = None,
- ) -> CompressionPlan:
- """Plan compression for time series data.
-
- Keeps items around change points (anomalies) plus first/last items.
- Uses STATISTICAL outlier detection for important items.
- Uses RelevanceScorer for semantic matching of user queries.
-
- Args:
- preserve_fields: TOIN-learned fields that users commonly retrieve.
- Items where query_context matches these field values get priority.
- max_items: Thread-safe max items limit (defaults to config value).
- """
- # Use provided max_items or fall back to config
- effective_max = max_items if max_items is not None else self.config.max_items_after_crush
- n = len(items)
- keep_indices = set()
-
- # 1. Dynamic anchor selection (replaces static first 3 + last 2)
- anchor_pattern = self._map_to_anchor_pattern(CompressionStrategy.TIME_SERIES)
- anchor_indices = self._anchor_selector.select_anchors(
- items=items,
- max_items=effective_max,
- pattern=anchor_pattern,
- query=query_context or None,
)
- keep_indices.update(anchor_indices)
-
- # 2. Items around change points from numeric fields
- for stats in analysis.field_stats.values():
- if stats.change_points:
- for cp in stats.change_points:
- # Keep a window around each change point
- for offset in range(-2, 3):
- idx = cp + offset
- if 0 <= idx < n:
- keep_indices.add(idx)
-
- # 3. Structural outlier items (STATISTICAL detection - no hardcoded keywords)
- outlier_indices = _detect_structural_outliers(items)
- keep_indices.update(outlier_indices)
-
- # 3b. Error items via KEYWORD detection (PRESERVATION GUARANTEE)
- # This is critical - errors must ALWAYS be preserved regardless of structure
- error_indices = _detect_error_items_for_preservation(items)
- keep_indices.update(error_indices)
-
- # 4. Items matching query anchors (DETERMINISTIC exact match)
- # Anchors provide reliable preservation for specific entity lookups (UUIDs, IDs, names)
- if query_context:
- anchors = extract_query_anchors(query_context)
- for i, item in enumerate(items):
- if item_matches_anchors(item, anchors):
- keep_indices.add(i)
-
- # 5. Items with high relevance to query context (PROBABILISTIC semantic match)
- if query_context:
- # Reuse pre-computed item_strings if available
- item_strs = (
- item_strings
- if item_strings is not None
- else [json.dumps(item, default=str) for item in items]
- )
- scores = self._scorer.score_batch(item_strs, query_context)
- for i, score in enumerate(scores):
- if score.score >= self._relevance_threshold:
- keep_indices.add(i)
- # 5b. TOIN preserve_fields: boost items where query matches these fields
- # Note: preserve_fields are SHA256[:8] hashes, use helper to match
- if preserve_fields and query_context:
- for i, item in enumerate(items):
- if _item_has_preserve_field_match(item, preserve_fields, query_context):
- keep_indices.add(i)
-
- # Limit to effective_max while ALWAYS preserving outliers and anomalies
- keep_indices = self._prioritize_indices(keep_indices, items, n, analysis, effective_max)
-
- plan.keep_indices = sorted(keep_indices)
- return plan
-
- def _plan_cluster_sample(
- self,
- analysis: ArrayAnalysis,
- items: list[dict],
- plan: CompressionPlan,
- query_context: str = "",
- preserve_fields: list[str] | None = None,
- max_items: int | None = None,
- item_strings: list[str] | None = None,
- ) -> CompressionPlan:
- """Plan compression for clusterable data (like logs).
-
- Uses clustering plus STATISTICAL outlier detection.
- Uses RelevanceScorer for semantic matching of user queries.
+ def crush(self, content: str, query: str = "", bias: float = 1.0) -> CrushResult:
+ """Crush a single JSON content string.
- Args:
- preserve_fields: TOIN-learned fields that users commonly retrieve.
- Items where query_context matches these field values get priority.
- max_items: Thread-safe max items limit (defaults to config value).
+ Mirrors the retired Python method. Returns a `CrushResult`
+ dataclass so call sites that destructure with `asdict()` keep
+ working.
"""
- # Use provided max_items or fall back to config
- effective_max = max_items if max_items is not None else self.config.max_items_after_crush
- n = len(items)
- keep_indices = set()
-
- # 1. Dynamic anchor selection (replaces static first 3 + last 2)
- anchor_pattern = self._map_to_anchor_pattern(CompressionStrategy.CLUSTER_SAMPLE)
- anchor_indices = self._anchor_selector.select_anchors(
- items=items,
- max_items=effective_max,
- pattern=anchor_pattern,
- query=query_context or None,
+ r = self._rust.crush(content, query, bias)
+ return CrushResult(
+ compressed=r.compressed,
+ original=r.original,
+ was_modified=r.was_modified,
+ strategy=r.strategy,
)
- keep_indices.update(anchor_indices)
-
- # 2. Structural outlier items (STATISTICAL detection - no hardcoded keywords)
- outlier_indices = _detect_structural_outliers(items)
- keep_indices.update(outlier_indices)
-
- # 2b. Error items via KEYWORD detection (PRESERVATION GUARANTEE)
- # This is critical - errors must ALWAYS be preserved regardless of structure
- error_indices = _detect_error_items_for_preservation(items)
- keep_indices.update(error_indices)
-
- # 3. Cluster by message-like field and keep representatives
- # Find a high-cardinality string field (likely message field)
- message_field = None
- max_uniqueness = 0.0
- for name, stats in analysis.field_stats.items():
- if stats.field_type == "string" and stats.unique_ratio > max_uniqueness:
- # Prefer fields with moderate to high uniqueness (message-like)
- if stats.unique_ratio > 0.3:
- message_field = name
- max_uniqueness = stats.unique_ratio
-
- if message_field:
- plan.cluster_field = message_field
-
- # Simple clustering: group by first 50 chars of message
- clusters: dict[str, list[int]] = {}
- for i, item in enumerate(items):
- msg = str(item.get(message_field, ""))[:50]
- msg_hash = hashlib.md5(msg.encode()).hexdigest()[:8] # nosec B324
- if msg_hash not in clusters:
- clusters[msg_hash] = []
- clusters[msg_hash].append(i)
-
- # Keep 1-2 representatives from each cluster
- for indices in clusters.values():
- for idx in indices[:2]:
- keep_indices.add(idx)
-
- # 4. Items matching query anchors (DETERMINISTIC exact match)
- # Anchors provide reliable preservation for specific entity lookups (UUIDs, IDs, names)
- if query_context:
- anchors = extract_query_anchors(query_context)
- for i, item in enumerate(items):
- if item_matches_anchors(item, anchors):
- keep_indices.add(i)
- # 5. Items with high relevance to query context (PROBABILISTIC semantic match)
- if query_context:
- # Reuse pre-computed item_strings if available
- item_strs = (
- item_strings
- if item_strings is not None
- else [json.dumps(item, default=str) for item in items]
- )
- scores = self._scorer.score_batch(item_strs, query_context)
- for i, score in enumerate(scores):
- if score.score >= self._relevance_threshold:
- keep_indices.add(i)
-
- # 5b. TOIN preserve_fields: boost items where query matches these fields
- # Note: preserve_fields are SHA256[:8] hashes, use helper to match
- if preserve_fields and query_context:
- for i, item in enumerate(items):
- if _item_has_preserve_field_match(item, preserve_fields, query_context):
- keep_indices.add(i)
-
- # Limit total while ALWAYS preserving outliers and anomalies
- keep_indices = self._prioritize_indices(keep_indices, items, n, analysis, effective_max)
-
- plan.keep_indices = sorted(keep_indices)
- return plan
-
- def _plan_top_n(
+ def _smart_crush_content(
self,
- analysis: ArrayAnalysis,
- items: list[dict],
- plan: CompressionPlan,
+ content: str,
query_context: str = "",
- preserve_fields: list[str] | None = None,
- max_items: int | None = None,
- item_strings: list[str] | None = None,
- ) -> CompressionPlan:
- """Plan compression for scored/ranked data.
-
- For data with a score/relevance field, that field IS the primary relevance
- signal. Our internal relevance scoring is SECONDARY - it's used to find
- potential "needle" items that the original scoring might have missed.
-
- Strategy:
- 1. Keep top N by score (the original system's relevance ranking)
- 2. Add structural outliers (errors, anomalies)
- 3. Add high-confidence relevance matches (needles the user is looking for)
+ tool_name: str | None = None,
+ bias: float = 1.0,
+ ) -> tuple[str, bool, str]:
+ """Apply smart crushing; return `(crushed, was_modified, info)`.
- Args:
- preserve_fields: TOIN-learned fields that users commonly retrieve.
- Items where query_context matches these field values get priority.
- max_items: Thread-safe max items limit (defaults to config value).
+ Mirrors the retired Python method's tuple shape. `tool_name` is
+ accepted for API compatibility and currently ignored — Stage
+ 3c.1 has no per-tool TOIN learning hook.
"""
- # Use provided max_items or fall back to config
- effective_max = max_items if max_items is not None else self.config.max_items_after_crush
-
- # Find score field using STATISTICAL detection (no hardcoded field names)
- score_field = None
- max_confidence = 0.0
- for name, stats in analysis.field_stats.items():
- is_score, confidence = _detect_score_field_statistically(stats, items)
- if is_score and confidence > max_confidence:
- score_field = name
- max_confidence = confidence
-
- if not score_field:
- return self._plan_smart_sample(
- analysis,
- items,
- plan,
- query_context,
- preserve_fields,
- effective_max,
- item_strings=item_strings,
- )
-
- plan.sort_field = score_field
- keep_indices = set()
+ crushed, was_modified, info = self._rust.smart_crush_content(content, query_context, bias)
+ return crushed, was_modified, info
- # 1. TOP N by score FIRST (the primary relevance signal)
- # The original system's score field is the authoritative ranking
- scored_items = [(i, item.get(score_field, 0)) for i, item in enumerate(items)]
- scored_items.sort(key=lambda x: x[1], reverse=True)
-
- # Reserve slots for outliers
- top_count = max(0, effective_max - 3)
- for idx, _ in scored_items[:top_count]:
- keep_indices.add(idx)
-
- # 2. Structural outlier items (STATISTICAL detection - no hardcoded keywords)
- outlier_indices = _detect_structural_outliers(items)
- keep_indices.update(outlier_indices)
+ def _extract_context_from_messages(self, messages: list[dict[str, Any]]) -> str:
+ """Build a query string from the last 5 user messages + recent
+ assistant tool-call arguments. Used by `apply()` to derive the
+ relevance context per-request.
- # 2b. Error items via KEYWORD detection (PRESERVATION GUARANTEE)
- # This is critical - errors must ALWAYS be preserved regardless of structure
- error_indices = _detect_error_items_for_preservation(items)
- keep_indices.update(error_indices)
+ Pure Python because it walks the message envelope, not the
+ compressed payload. The retired implementation lived inline on
+ `SmartCrusher`; preserved here unchanged.
+ """
+ context_parts: list[str] = []
+ user_message_count = 0
- # 3. Items matching query anchors (DETERMINISTIC exact match) - ADDITIVE
- # Anchors provide reliable preservation for specific entity lookups (UUIDs, IDs, names)
- # These are ALWAYS preserved since they represent explicit user intent
- if query_context:
- anchors = extract_query_anchors(query_context)
- for i, item in enumerate(items):
- if i not in keep_indices and item_matches_anchors(item, anchors):
- keep_indices.add(i)
+ for msg in reversed(messages):
+ if msg.get("role") == "user":
+ content = msg.get("content")
+ if isinstance(content, str):
+ context_parts.append(content)
+ elif isinstance(content, list):
+ for block in content:
+ if isinstance(block, dict) and block.get("type") == "text":
+ text = block.get("text", "")
+ if text:
+ context_parts.append(text)
- # 4. HIGH-CONFIDENCE relevance matches (potential needles) - ADDITIVE only
- # Only add items that are NOT already in top N but match the query strongly
- # Use a higher threshold (0.5) since the score field already captures relevance
- if query_context:
- # Reuse pre-computed item_strings if available
- item_strs = (
- item_strings
- if item_strings is not None
- else [json.dumps(item, default=str) for item in items]
- )
- scores = self._scorer.score_batch(item_strs, query_context)
- # Higher threshold and limit count to avoid adding everything
- high_threshold = max(0.5, self._relevance_threshold * 2)
- added_count = 0
- max_relevance_adds = 3 # Limit additional relevance matches
- for i, score in enumerate(scores):
- if i not in keep_indices and score.score >= high_threshold:
- keep_indices.add(i)
- added_count += 1
- if added_count >= max_relevance_adds:
- break
+ user_message_count += 1
+ if user_message_count >= 5:
+ break
- # 4b. TOIN preserve_fields: boost items where query matches these fields
- # Note: preserve_fields are SHA256[:8] hashes, use helper to match
- if preserve_fields and query_context:
- for i, item in enumerate(items):
- if i not in keep_indices: # Only add if not already kept
- if _item_has_preserve_field_match(item, preserve_fields, query_context):
- keep_indices.add(i)
+ if msg.get("role") == "assistant" and msg.get("tool_calls"):
+ for tc in msg.get("tool_calls", []):
+ if isinstance(tc, dict):
+ func = tc.get("function", {})
+ args = func.get("arguments", "")
+ if isinstance(args, str) and args:
+ context_parts.append(args)
- plan.keep_count = len(keep_indices)
- plan.keep_indices = sorted(keep_indices)
- return plan
+ return " ".join(context_parts)
- def _plan_smart_sample(
+ def apply(
self,
- analysis: ArrayAnalysis,
- items: list[dict],
- plan: CompressionPlan,
- query_context: str = "",
- preserve_fields: list[str] | None = None,
- max_items: int | None = None,
- item_strings: list[str] | None = None,
- ) -> CompressionPlan:
- """Plan smart statistical sampling using STATISTICAL detection.
-
- Always keeps:
- - Dynamic anchor positions (based on data pattern and query context)
- - Structural outliers (items with rare fields or rare status values)
- - Anomalous numeric items (> 2 std from mean)
- - Items around change points
- - Items with high relevance to query context (via RelevanceScorer)
-
- Uses STATISTICAL detection instead of hardcoded keywords.
+ messages: list[dict[str, Any]],
+ tokenizer: Tokenizer,
+ **kwargs: Any,
+ ) -> TransformResult:
+ """Transform-protocol entry point. Walks every tool/tool_result
+ message, applies SmartCrusher to large enough payloads, and
+ replaces the message content with `<crushed>\\n<digest_marker>`.
- Args:
- preserve_fields: TOIN-learned fields that users commonly retrieve.
- Items where query_context matches these field values get priority.
- max_items: Thread-safe max items limit (defaults to config value).
+ Pure orchestration — the per-message compression delegates to
+ Rust via `_smart_crush_content`.
"""
- # Use provided max_items or fall back to config
- effective_max = max_items if max_items is not None else self.config.max_items_after_crush
-
- n = len(items)
- keep_indices = set()
-
- # 1. Dynamic anchor selection (replaces static first 3 + last 2)
- anchor_pattern = self._map_to_anchor_pattern(CompressionStrategy.SMART_SAMPLE)
- anchor_indices = self._anchor_selector.select_anchors(
- items=items,
- max_items=effective_max,
- pattern=anchor_pattern,
- query=query_context or None,
- )
- keep_indices.update(anchor_indices)
-
- # 2. Structural outlier items (STATISTICAL detection - no hardcoded keywords)
- outlier_indices = _detect_structural_outliers(items)
- keep_indices.update(outlier_indices)
-
- # 2b. Error items via KEYWORD detection (PRESERVATION GUARANTEE)
- # This is critical - errors must ALWAYS be preserved regardless of structure
- error_indices = _detect_error_items_for_preservation(items)
- keep_indices.update(error_indices)
-
- # 3. Anomalous numeric items (> 2 std from mean)
- for name, stats in analysis.field_stats.items():
- if stats.field_type == "numeric" and stats.mean_val is not None and stats.variance:
- std = stats.variance**0.5
- if std > 0:
- threshold = self.config.variance_threshold * std
- for i, item in enumerate(items):
- val = item.get(name)
- if isinstance(val, int | float):
- if abs(val - stats.mean_val) > threshold:
- keep_indices.add(i)
-
- # 4. Items around change points (if detected)
- if self.config.preserve_change_points:
- for stats in analysis.field_stats.values():
- if stats.change_points:
- for cp in stats.change_points:
- # Keep items around change point
- for offset in range(-1, 2):
- idx = cp + offset
- if 0 <= idx < n:
- keep_indices.add(idx)
+ tokens_before = tokenizer.count_messages(messages)
+ result_messages = deep_copy_messages(messages)
+ transforms_applied: list[str] = []
+ markers_inserted: list[str] = []
+ warnings: list[str] = []
- # 5. Items matching query anchors (DETERMINISTIC exact match)
- # Anchors provide reliable preservation for specific entity lookups (UUIDs, IDs, names)
- if query_context:
- anchors = extract_query_anchors(query_context)
- for i, item in enumerate(items):
- if item_matches_anchors(item, anchors):
- keep_indices.add(i)
+ query_context = self._extract_context_from_messages(result_messages)
+ crushed_count = 0
+ frozen_message_count = kwargs.get("frozen_message_count", 0)
- # 6. Items with high relevance to query context (PROBABILISTIC semantic match)
- if query_context:
- # Reuse pre-computed item_strings if available
- item_strs = (
- item_strings
- if item_strings is not None
- else [json.dumps(item, default=str) for item in items]
- )
- scores = self._scorer.score_batch(item_strs, query_context)
- for i, score in enumerate(scores):
- if score.score >= self._relevance_threshold:
- keep_indices.add(i)
+ for msg_idx, msg in enumerate(result_messages):
+ if msg_idx < frozen_message_count:
+ continue
- # 6b. TOIN preserve_fields: boost items where query matches these fields
- # Note: preserve_fields are SHA256[:8] hashes, use helper to match
- if preserve_fields and query_context:
- for i, item in enumerate(items):
- if _item_has_preserve_field_match(item, preserve_fields, query_context):
- keep_indices.add(i)
+ # OpenAI-style: top-level role=tool with string content.
+ if msg.get("role") == "tool":
+ content = msg.get("content", "")
+ if isinstance(content, str):
+ tokens = tokenizer.count_text(content)
+ if tokens > self.config.min_tokens_to_crush:
+ crushed, was_modified, info = self._smart_crush_content(
+ content, query_context
+ )
+ if was_modified:
+ marker = create_tool_digest_marker(compute_short_hash(content))
+ msg["content"] = crushed + "\n" + marker
+ crushed_count += 1
+ markers_inserted.append(marker)
+ if info:
+ transforms_applied.append(f"smart:{info}")
+
+ # Anthropic-style: content is a list of blocks; each tool_result
+ # block has a string content field of its own.
+ content = msg.get("content")
+ if isinstance(content, list):
+ for i, block in enumerate(content):
+ if not isinstance(block, dict) or block.get("type") != "tool_result":
+ continue
+ tool_content = block.get("content", "")
+ if not isinstance(tool_content, str):
+ continue
+ tokens = tokenizer.count_text(tool_content)
+ if tokens <= self.config.min_tokens_to_crush:
+ continue
- # Limit to effective_max while ALWAYS preserving outliers and anomalies
- keep_indices = self._prioritize_indices(keep_indices, items, n, analysis, effective_max)
+ crushed, was_modified, info = self._smart_crush_content(
+ tool_content, query_context
+ )
+ if was_modified:
+ marker = create_tool_digest_marker(compute_short_hash(tool_content))
+ content[i]["content"] = crushed + "\n" + marker
+ crushed_count += 1
+ markers_inserted.append(marker)
+ if info:
+ transforms_applied.append(f"smart:{info}")
- plan.keep_indices = sorted(keep_indices)
- return plan
+ if crushed_count > 0:
+ transforms_applied.insert(0, f"smart_crush:{crushed_count}")
- def _execute_plan(
- self, plan: CompressionPlan, items: list[dict], analysis: ArrayAnalysis
- ) -> list:
- """Execute a compression plan and return crushed array.
+ tokens_after = tokenizer.count_messages(result_messages)
- SCHEMA-PRESERVING: Returns only items from the original array.
- No wrappers, no generated text, no metadata keys.
- """
- result = []
+ return TransformResult(
+ messages=result_messages,
+ tokens_before=tokens_before,
+ tokens_after=tokens_after,
+ transforms_applied=transforms_applied,
+ markers_inserted=markers_inserted,
+ warnings=warnings,
+ )
- # Return only the kept items, preserving original schema
- for idx in sorted(plan.keep_indices):
- if 0 <= idx < len(items):
- # Copy item unchanged - no modifications to schema
- result.append(items[idx].copy())
- return result
+# ─── Convenience function ─────────────────────────────────────────────────
def smart_crush_tool_output(
@@ -3719,32 +331,12 @@ def smart_crush_tool_output(
config: SmartCrusherConfig | None = None,
ccr_config: CCRConfig | None = None,
) -> tuple[str, bool, str]:
- """
- Convenience function to smart-crush a single tool output.
-
- NOTE: CCR markers are DISABLED by default in this convenience function
- to maintain backward compatibility (output remains valid JSON).
- To enable CCR markers, pass a CCRConfig with inject_retrieval_marker=True.
+ """Compress a single tool output. Returns `(crushed, was_modified, info)`.
- Args:
- content: The tool output content (JSON string).
- config: Optional SmartCrusher configuration.
- ccr_config: Optional CCR (Compress-Cache-Retrieve) configuration.
- By default, CCR is enabled (caching) but markers are disabled.
-
- Returns:
- Tuple of (crushed_content, was_modified, analysis_info).
+ Convenience wrapper that builds a one-shot `SmartCrusher` per call.
+ `ccr_config` is accepted for source compatibility but currently
+ not wired through (Stage 3c.1 keeps CCR marker injection
+ disabled — Rust port has no compression store).
"""
- cfg = config or SmartCrusherConfig()
-
- # Default: CCR enabled for caching, but markers disabled for clean JSON output
- if ccr_config is None:
- ccr_cfg = CCRConfig(
- enabled=True, # Still cache for retrieval
- inject_retrieval_marker=False, # Don't break JSON output
- )
- else:
- ccr_cfg = ccr_config
-
- crusher = SmartCrusher(cfg, ccr_config=ccr_cfg)
+ crusher = SmartCrusher(config=config, ccr_config=ccr_config)
return crusher._smart_crush_content(content)
diff --git a/tests/test_acceptance.py b/tests/test_acceptance.py
index 6a4d3e680..c8636c084 100644
--- a/tests/test_acceptance.py
+++ b/tests/test_acceptance.py
@@ -447,11 +447,7 @@ class TestQueryAnchorExtraction:
"""If user asks for 'Alice', item with Alice should be preserved."""
import json
- from headroom.transforms.smart_crusher import (
- SmartCrusher,
- SmartCrusherConfig,
- extract_query_anchors,
- )
+ from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig
# User is searching for 'Alice'
messages = [
@@ -478,38 +474,31 @@ class TestQueryAnchorExtraction:
},
]
- # Verify anchor extraction works
- anchors = extract_query_anchors("Find the user named 'Alice' in the system.")
- assert "alice" in anchors
-
- # Verify crushing preserves Alice
+ # End-to-end behavior: the relevance scorer (HybridScorer in
+ # the Rust port — BM25 + embedding) should pick up "Alice"
+ # from the user message and preserve the matching tool item
+ # even though it sits at index 50.
config = SmartCrusherConfig(
enabled=True,
min_items_to_analyze=5,
min_tokens_to_crush=100,
- max_items_after_crush=10, # Should normally drop Alice at index 50
+ max_items_after_crush=10,
)
crusher = SmartCrusher(config)
tokenizer = get_tokenizer()
result = crusher.apply(messages, tokenizer)
- # Find the crushed tool output
tool_msg = next(m for m in result.messages if m.get("role") == "tool")
crushed_content = tool_msg["content"]
- # Alice should be preserved even though she's at index 50
assert "Alice" in crushed_content
def test_preserves_needle_by_uuid(self):
"""If user asks for a UUID, item with that UUID should be preserved."""
import json
- from headroom.transforms.smart_crusher import (
- SmartCrusher,
- SmartCrusherConfig,
- extract_query_anchors,
- )
+ from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig
target_uuid = "550e8400-e29b-41d4-a716-446655440000"
@@ -537,10 +526,6 @@ class TestQueryAnchorExtraction:
},
]
- # Verify anchor extraction
- anchors = extract_query_anchors(f"Get details for request {target_uuid}")
- assert target_uuid.lower() in anchors
-
config = SmartCrusherConfig(
enabled=True,
min_items_to_analyze=5,
@@ -555,7 +540,6 @@ class TestQueryAnchorExtraction:
tool_msg = next(m for m in result.messages if m.get("role") == "tool")
crushed_content = tool_msg["content"]
- # UUID should be preserved
assert target_uuid in crushed_content
diff --git a/tests/test_ccr.py b/tests/test_ccr.py
index d11454a0c..c68880725 100644
--- a/tests/test_ccr.py
+++ b/tests/test_ccr.py
@@ -235,160 +235,6 @@ class TestCompressionStore:
assert store.exists(h)
-class TestSmartCrusherCCRIntegration:
- """Test SmartCrusher integration with CCR."""
-
- @pytest.fixture(autouse=True)
- def reset_store(self):
- """Reset global store before each test."""
- reset_compression_store()
- yield
- reset_compression_store()
-
- def test_compression_caches_original(self):
- """SmartCrusher caches original content when compressing."""
- items = [{"id": i, "score": 100 - i, "data": f"item_{i}"} for i in range(100)]
- content = json.dumps(items)
-
- config = SmartCrusherConfig(max_items_after_crush=15)
- ccr_config = CCRConfig(
- enabled=True,
- inject_retrieval_marker=False, # Don't add marker for this test
- min_items_to_cache=10,
- )
-
- compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config)
-
- assert was_modified
-
- # Check that content was cached
- store = get_compression_store()
- stats = store.get_stats()
- assert stats["entry_count"] >= 1
-
- def test_retrieval_marker_injected(self):
- """CCR marker is injected when configured."""
- items = [{"id": i, "score": 100 - i, "data": f"item_{i}"} for i in range(100)]
- content = json.dumps(items)
-
- config = SmartCrusherConfig(max_items_after_crush=15)
- ccr_config = CCRConfig(
- enabled=True,
- inject_retrieval_marker=True,
- min_items_to_cache=10,
- )
-
- compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config)
-
- assert was_modified
- # Marker should be present
- assert "items compressed" in compressed_str or "hash=" in compressed_str
-
- def test_small_arrays_not_cached(self):
- """Arrays smaller than min_items_to_cache are not cached."""
- items = [{"id": i} for i in range(15)]
- content = json.dumps(items)
-
- config = SmartCrusherConfig(max_items_after_crush=10)
- ccr_config = CCRConfig(
- enabled=True,
- min_items_to_cache=50, # Require 50+ items
- )
-
- smart_crush_tool_output(content, config, ccr_config)
-
- store = get_compression_store()
- stats = store.get_stats()
- # Should not cache because original has < 50 items
- assert stats["entry_count"] == 0
-
- def test_uncrushed_data_not_cached(self):
- """Data that doesn't get crushed is not cached."""
- # DB results with unique IDs - shouldn't be crushed
- items = [{"id": i, "name": f"User {i}", "email": f"user{i}@test.com"} for i in range(30)]
- content = json.dumps(items)
-
- config = SmartCrusherConfig(max_items_after_crush=10)
- ccr_config = CCRConfig(enabled=True, min_items_to_cache=10)
-
- compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config)
-
- # If not modified, shouldn't be cached
- if not was_modified:
- store = get_compression_store()
- stats = store.get_stats()
- assert stats["entry_count"] == 0
-
- def test_can_retrieve_after_compression(self):
- """Can retrieve original content after compression."""
- items = [
- {"id": i, "score": 100 - i, "content": f"Document about topic {i}"} for i in range(100)
- ]
- content = json.dumps(items)
-
- config = SmartCrusherConfig(max_items_after_crush=15)
- ccr_config = CCRConfig(
- enabled=True,
- inject_retrieval_marker=True,
- min_items_to_cache=10,
- )
-
- compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config)
-
- assert was_modified
-
- # Extract hash from marker
- # Marker format: [100 items compressed to 15. Retrieve more: hash=abc123...]
- import re
-
- match = re.search(r"hash=([a-f0-9]+)", compressed_str)
- assert match is not None, f"No hash found in: {compressed_str}"
-
- hash_key = match.group(1)
-
- # Retrieve original
- store = get_compression_store()
- entry = store.retrieve(hash_key)
-
- assert entry is not None
- original_items = json.loads(entry.original_content)
- assert len(original_items) == 100
-
- def test_search_after_compression(self):
- """Can search within original content after compression."""
- items = [
- {"id": 1, "content": "Authentication error: invalid token"},
- {"id": 2, "content": "Database connection successful"},
- {"id": 3, "content": "User login completed"},
- ] + [{"id": i, "content": f"Generic log entry {i}"} for i in range(4, 104)]
- content = json.dumps(items)
-
- config = SmartCrusherConfig(max_items_after_crush=15)
- ccr_config = CCRConfig(
- enabled=True,
- inject_retrieval_marker=True,
- min_items_to_cache=10,
- )
-
- compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config)
-
- assert was_modified
-
- # Extract hash
- import re
-
- match = re.search(r"hash=([a-f0-9]+)", compressed_str)
- hash_key = match.group(1)
-
- # Search for authentication items
- store = get_compression_store()
- results = store.search(hash_key, "authentication error token")
-
- # Should find the authentication error item
- assert len(results) >= 1
- assert any("Authentication" in r.get("content", "") for r in results)
-
-
class TestCCRConfig:
"""Test CCR configuration options."""
@@ -402,24 +248,6 @@ class TestCCRConfig:
assert config.feedback_enabled is True
assert config.min_items_to_cache == 20
- def test_custom_marker_template(self):
- """Custom marker template is used."""
- items = [{"id": i, "score": 100 - i} for i in range(100)]
- content = json.dumps(items)
-
- config = SmartCrusherConfig(max_items_after_crush=15)
- ccr_config = CCRConfig(
- enabled=True,
- inject_retrieval_marker=True,
- min_items_to_cache=10,
- marker_template="\n[CUSTOM: {original_count} -> {compressed_count}, key={hash}]",
- )
-
- compressed_str, was_modified, _ = smart_crush_tool_output(content, config, ccr_config)
-
- if was_modified:
- assert "CUSTOM:" in compressed_str or "key=" in compressed_str
-
class TestCCRFeedbackLoop:
"""Test CCR feedback tracking for learning."""
diff --git a/tests/test_critical_fixes.py b/tests/test_critical_fixes.py
index fd6d8263f..c00a7d0e5 100644
--- a/tests/test_critical_fixes.py
+++ b/tests/test_critical_fixes.py
@@ -11,8 +11,6 @@ These tests verify the before/after behavior of critical bug fixes:
import time
from unittest.mock import patch
-import pytest
-
class TestTOINConfidenceMathFix:
"""Test for CRITICAL: Confidence calculation math error in toin.py:721.
@@ -303,146 +301,5 @@ class TestUnboundedStrategyDicts:
)
-class TestSmartCrusherTOINIntegration:
- """Test for CRITICAL: SmartCrusher not calling toin.record_compression().
-
- BUG: SmartCrusher calls feedback.record_compression() but never calls
- toin.record_compression(). This means TOIN only learns from retrieval events,
- not from compression events - breaking the feedback loop.
-
- FIX: Add toin.record_compression() call after compression in SmartCrusher.
- """
-
- def test_smart_crusher_records_to_toin(self):
- """SmartCrusher should record compression events to TOIN."""
- from headroom.telemetry.models import ToolSignature
- from headroom.telemetry.toin import get_toin, reset_toin
- from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig
-
- reset_toin()
-
- config = SmartCrusherConfig(
- min_items_to_analyze=5,
- max_items_after_crush=10,
- use_feedback_hints=True,
- )
- crusher = SmartCrusher(config)
-
- # Create test items that look like search results with a clear score field
- # This pattern is crushable because:
- # 1. Has a clear numeric score field in BOUNDED range [0,1]
- # 2. Has repeated structure with some constant fields (type, language)
- # 3. Score values vary within the bounded range
- items = [
- {
- "name": f"repo_{i}",
- "relevance_score": (50 - i) / 50.0, # Bounded [0,1] - descending order
- "type": "repository", # Constant field
- "language": "python" if i % 3 == 0 else "javascript", # Low cardinality
- "description": f"Description {i % 5}", # Low cardinality
- }
- for i in range(50)
- ]
-
- # Get TOIN instance and check initial state
- toin = get_toin()
- len(toin._patterns)
-
- # Crush the array
- result, info, markers, _summary = crusher._crush_array(
- items, query_context="test query", tool_name="test_tool"
- )
-
- # Verify compression happened (not skipped)
- assert "skip" not in info.lower(), (
- f"Compression was skipped: {info}. Test needs crushable data."
- )
-
- # Get the signature that would have been created
- sig = ToolSignature.from_items(items)
-
- # Check TOIN was notified
- with toin._lock:
- pattern = toin._patterns.get(sig.structure_hash)
-
- # After fix, TOIN should have a pattern for this tool's signature
- assert pattern is not None, (
- f"TOIN should have recorded the compression event. "
- f"Info: {info}, pattern count: {len(toin._patterns)}"
- )
- if pattern:
- assert pattern.total_compressions >= 1, (
- f"Pattern should have at least 1 compression recorded, got {pattern.total_compressions}"
- )
-
-
class TestAllFixesIntegrated:
"""Integration tests ensuring all fixes work together."""
-
- def test_full_feedback_loop(self):
- """Test complete feedback loop: compress -> store -> retrieve -> learn."""
- from headroom.cache.compression_feedback import (
- reset_compression_feedback,
- )
- from headroom.cache.compression_store import reset_compression_store
- from headroom.telemetry.models import ToolSignature
- from headroom.telemetry.toin import get_toin, reset_toin
- from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig
-
- # Reset all singletons
- reset_toin()
- reset_compression_store()
- reset_compression_feedback()
-
- # Setup
- config = SmartCrusherConfig(
- min_items_to_analyze=5,
- max_items_after_crush=10,
- use_feedback_hints=True,
- )
- crusher = SmartCrusher(config)
-
- # Create test items that look like API responses with scoring
- # This pattern is crushable because:
- # 1. Has a clear numeric score field in BOUNDED range [0,1]
- # 2. Has constant fields (status, type)
- # 3. Has enough items for compression (100)
- items = [
- {
- "priority": (100 - i) / 100.0, # Bounded [0,1] - descending order
- "status": "ok", # Constant field
- "type": "response", # Constant field
- "data": f"content_{i % 10}", # Low cardinality (only 10 unique values)
- }
- for i in range(100)
- ]
-
- # Step 1: Compress
- result, info, markers, _summary = crusher._crush_array(
- items, query_context="find status", tool_name="api_response"
- )
-
- # Verify compression happened (not skipped)
- assert "skip" not in info.lower(), (
- f"Compression was skipped: {info}. Test needs crushable data."
- )
-
- # Step 2: Check TOIN was notified (after fix)
- toin = get_toin()
- sig = ToolSignature.from_items(items)
-
- with toin._lock:
- toin_pattern = toin._patterns.get(sig.structure_hash)
-
- # After fix, TOIN should have the pattern
- assert toin_pattern is not None, (
- f"TOIN should have learned from the compression event. Info: {info}"
- )
- assert toin_pattern.total_compressions >= 1, (
- f"TOIN pattern should have recorded compression, got {toin_pattern.total_compressions}"
- )
-
-
-# Run specific test to verify fix
-if __name__ == "__main__":
- pytest.main([__file__, "-v", "--tb=short"])
diff --git a/tests/test_crushability.py b/tests/test_crushability.py
deleted file mode 100644
index e67ac428b..000000000
--- a/tests/test_crushability.py
+++ /dev/null
@@ -1,423 +0,0 @@
-"""Tests for SmartCrusher crushability analysis.
-
-These tests verify that SmartCrusher correctly identifies when it's SAFE
-to crush data vs when it should SKIP crushing.
-
-The key insight: High variability + No importance signal = DON'T CRUSH.
-
-Test scenarios:
-1. DB results (unique entities, no signal) → SKIP
-2. Search results (has score field) → CRUSH using score
-3. Log entries (has errors) → CRUSH keeping errors
-4. Time series (has anomalies) → CRUSH keeping anomalies
-5. Repetitive data (low uniqueness) → CRUSH with sampling
-"""
-
-import json
-
-import pytest
-
-from headroom.transforms.smart_crusher import (
- CompressionStrategy,
- SmartAnalyzer,
- SmartCrusherConfig,
- smart_crush_tool_output,
-)
-
-
-class TestCrushabilityDetection:
- """Test the crushability analysis logic."""
-
- @pytest.fixture
- def analyzer(self):
- """Create a SmartAnalyzer instance."""
- return SmartAnalyzer(SmartCrusherConfig())
-
- def test_db_results_not_crushable(self, analyzer):
- """DB query results with unique IDs and no signal should NOT be crushed."""
- # Simulate: SELECT * FROM users LIMIT 50
- items = [
- {
- "id": i,
- "name": f"User {i}",
- "email": f"user{i}@example.com",
- "department": "Engineering",
- }
- for i in range(50)
- ]
-
- analysis = analyzer.analyze_array(items)
-
- # Should detect unique entities with no importance signal
- assert analysis.crushability is not None
- assert not analysis.crushability.crushable, (
- f"DB results should NOT be crushable. "
- f"Reason: {analysis.crushability.reason}, "
- f"Signals: {analysis.crushability.signals_present}"
- )
- assert analysis.recommended_strategy == CompressionStrategy.SKIP
- assert "unique" in analysis.crushability.reason.lower()
-
- def test_db_results_with_unique_uuid(self, analyzer):
- """DB results with UUID field should NOT be crushed."""
- items = [
- {
- "uuid": f"550e8400-e29b-41d4-a716-44665544{i:04d}",
- "name": f"Record {i}",
- "value": i * 10,
- }
- for i in range(50)
- ]
-
- analysis = analyzer.analyze_array(items)
-
- assert analysis.crushability is not None
- assert not analysis.crushability.crushable
- assert analysis.crushability.has_id_field
-
- def test_search_results_crushable(self, analyzer):
- """Search results with score field SHOULD be crushed."""
- items = [
- {
- "id": i,
- "title": f"Document {i}",
- "snippet": f"This is document {i} content...",
- "score": 1.0 - (i * 0.01), # Decreasing relevance
- }
- for i in range(100)
- ]
-
- analysis = analyzer.analyze_array(items)
-
- # Should detect score field as importance signal
- assert analysis.crushability is not None
- assert analysis.crushability.crushable, (
- f"Search results should be crushable. Reason: {analysis.crushability.reason}"
- )
- assert analysis.crushability.has_score_field
- assert any("score" in s for s in analysis.crushability.signals_present)
-
- def test_log_entries_with_errors_crushable(self, analyzer):
- """Log entries containing structural outliers SHOULD be crushed (outliers preserved)."""
- items = []
- for i in range(100):
- item = {
- "id": i,
- "timestamp": f"2024-01-15T10:{i:02d}:00Z",
- "message": f"Request processed successfully - {i}",
- "level": "INFO",
- }
- # Add some errors - these are STRUCTURAL OUTLIERS (have extra "error" field)
- if i % 20 == 0:
- item["level"] = "ERROR"
- item["message"] = f"Connection failed: timeout at {i}"
- item["error"] = "TimeoutError" # Extra field that most items don't have
- items.append(item)
-
- analysis = analyzer.analyze_array(items)
-
- # Should detect structural outliers (items with rare fields like "error")
- assert analysis.crushability is not None
- assert analysis.crushability.crushable
- # Now uses structural_outliers instead of keyword-based error count
- assert any(
- "structural_outliers" in s or "outlier" in s.lower()
- for s in analysis.crushability.signals_present
- )
-
- def test_time_series_with_anomalies_crushable(self, analyzer):
- """Time series with numeric anomalies SHOULD be crushed."""
- items = []
- for i in range(100):
- value = 100.0 # Normal value
- if i in [25, 50, 75]: # Anomaly points
- value = 999.0
- items.append(
- {
- "id": i,
- "timestamp": i,
- "cpu_usage": value,
- }
- )
-
- analysis = analyzer.analyze_array(items)
-
- # Should detect anomalies as importance signal
- assert analysis.crushability is not None
- assert analysis.crushability.crushable
- assert analysis.crushability.anomaly_count > 0
-
- def test_repetitive_data_crushable(self, analyzer):
- """Repetitive data (low uniqueness) SHOULD be crushable."""
- # Same status repeated many times
- items = [
- {
- "id": i,
- "status": "success", # Same for all
- "code": 200, # Same for all
- "message": "OK", # Same for all
- }
- for i in range(100)
- ]
-
- analysis = analyzer.analyze_array(items)
-
- # Should detect low uniqueness - safe to sample
- assert analysis.crushability is not None
- assert analysis.crushability.crushable
- # Can be "low_uniqueness" or "repetitive_content_with_ids"
- assert (
- "low_uniqueness" in analysis.crushability.reason
- or "repetitive" in analysis.crushability.reason
- )
-
- def test_file_listing_not_crushable(self, analyzer):
- """File listing with unique paths should NOT be crushed."""
- items = [
- {
- "id": i,
- "path": f"/home/user/project/src/module{i}/file{i}.py",
- "size": 1000 + i,
- "modified": f"2024-01-{(i % 28) + 1:02d}",
- }
- for i in range(50)
- ]
-
- analysis = analyzer.analyze_array(items)
-
- # Paths are highly unique, no importance signal
- assert analysis.crushability is not None
- # Should NOT crush file listings
- assert not analysis.crushability.crushable or analysis.crushability.confidence < 0.7
-
- def test_order_list_not_crushable(self, analyzer):
- """Order list with unique order IDs should NOT be crushed."""
- items = [
- {
- "order_id": f"ORD-2024-{i:05d}",
- "customer": f"Customer {i}",
- "total": 50.0 + i,
- "status": "completed",
- }
- for i in range(50)
- ]
-
- analysis = analyzer.analyze_array(items)
-
- # Each order is a unique entity
- assert analysis.crushability is not None
- # order_id contains 'id' pattern
- assert not analysis.crushability.crushable
-
-
-class TestCrushabilityEndToEnd:
- """End-to-end tests for crushability-aware crushing."""
-
- def test_db_results_preserved_completely(self):
- """DB results should be returned unchanged when not crushable."""
- items = [{"id": i, "name": f"User {i}", "email": f"user{i}@test.com"} for i in range(30)]
- content = json.dumps(items)
-
- config = SmartCrusherConfig(max_items_after_crush=10)
- crushed, was_modified, info = smart_crush_tool_output(content, config)
-
- # Should NOT be modified (skip crushing)
- if was_modified:
- result = json.loads(crushed)
- # If it was modified, all items should still be there
- assert len(result) == 30, (
- f"DB results should not lose items! Had 30, got {len(result)}. Info: {info}"
- )
-
- def test_search_results_crushed_by_score(self):
- """Search results should be crushed using score field."""
- items = [
- {
- "id": i,
- "title": f"Result {i}",
- "score": 100 - i, # Higher score = more relevant
- }
- for i in range(100)
- ]
- content = json.dumps(items)
-
- config = SmartCrusherConfig(max_items_after_crush=15)
- crushed, was_modified, info = smart_crush_tool_output(content, config)
-
- assert was_modified
- result = json.loads(crushed)
- assert len(result) < 100
-
- # Top scores should be preserved
- scores = [item.get("score", 0) for item in result]
- assert max(scores) >= 90 # Top items preserved
-
- def test_mixed_data_with_errors_preserves_errors(self):
- """Data with errors should crush but preserve ALL errors."""
- items = []
- error_ids = [5, 25, 45, 65, 85]
- for i in range(100):
- item = {"id": i, "data": f"value_{i}"}
- if i in error_ids:
- item["status"] = "failed"
- item["error"] = f"Error at {i}"
- items.append(item)
-
- content = json.dumps(items)
- config = SmartCrusherConfig(max_items_after_crush=20)
- crushed, was_modified, info = smart_crush_tool_output(content, config)
-
- result = json.loads(crushed)
-
- # All errors must be preserved
- error_count = sum(1 for item in result if item.get("error"))
- assert error_count == len(error_ids), (
- f"All {len(error_ids)} errors should be preserved, got {error_count}"
- )
-
-
-class TestCrushabilitySignals:
- """Test individual signal detection."""
-
- @pytest.fixture
- def analyzer(self):
- return SmartAnalyzer(SmartCrusherConfig())
-
- def test_detects_id_field_variations(self, analyzer):
- """Should detect various ID field naming patterns."""
- test_cases = [
- ("id", [{"id": i} for i in range(20)]),
- ("uuid", [{"uuid": f"uuid-{i}"} for i in range(20)]),
- ("_id", [{"_id": f"mongo-{i}"} for i in range(20)]),
- ("pk", [{"pk": i} for i in range(20)]),
- ("key", [{"key": f"key-{i}"} for i in range(20)]),
- ("user_id", [{"user_id": i} for i in range(20)]),
- ]
-
- for field_name, items in test_cases:
- analysis = analyzer.analyze_array(items)
- assert analysis.crushability is not None
- assert analysis.crushability.has_id_field, f"Should detect '{field_name}' as ID field"
-
- def test_detects_score_field_variations(self, analyzer):
- """Should detect various score field naming patterns."""
- test_cases = [
- "score",
- "rank",
- "relevance",
- "confidence",
- "_score",
- "rating",
- ]
-
- for field_name in test_cases:
- items = [{field_name: i * 0.1, "data": f"item_{i}"} for i in range(20)]
- analysis = analyzer.analyze_array(items)
- assert analysis.crushability is not None
- assert analysis.crushability.has_score_field, (
- f"Should detect '{field_name}' as score field"
- )
-
- def test_detects_error_keywords(self, analyzer):
- """Should detect various error keyword patterns."""
- error_keywords = ["error", "exception", "failed", "failure", "critical", "fatal"]
-
- for keyword in error_keywords:
- items = [{"id": i, "msg": "OK"} for i in range(20)]
- items[10]["msg"] = f"Something {keyword} happened"
-
- analysis = analyzer.analyze_array(items)
- assert analysis.crushability is not None
- assert analysis.crushability.error_item_count >= 1, (
- f"Should detect '{keyword}' as error indicator"
- )
-
-
-class TestCrushabilityEdgeCases:
- """Test edge cases in crushability analysis."""
-
- @pytest.fixture
- def analyzer(self):
- return SmartAnalyzer(SmartCrusherConfig())
-
- def test_empty_array(self, analyzer):
- """Empty array should not crash."""
- analysis = analyzer.analyze_array([])
- assert analysis.recommended_strategy == CompressionStrategy.NONE
-
- def test_small_array_skipped(self, analyzer):
- """Arrays below min_items_to_analyze should be skipped."""
- items = [{"id": i} for i in range(3)]
- analysis = analyzer.analyze_array(items)
- assert analysis.recommended_strategy == CompressionStrategy.NONE
-
- def test_mixed_signals(self, analyzer):
- """Data with multiple signals should still be crushable."""
- items = []
- for i in range(100):
- item = {
- "id": i,
- "score": 100 - i, # Score signal
- "value": 50.0,
- }
- if i == 50:
- item["error"] = "Test error" # Error signal
- item["value"] = 999.0 # Anomaly signal
- items.append(item)
-
- analysis = analyzer.analyze_array(items)
- assert analysis.crushability is not None
- assert analysis.crushability.crushable
- assert len(analysis.crushability.signals_present) >= 2
-
- def test_all_items_are_errors(self, analyzer):
- """When all items are errors, keyword detection finds them as a signal.
-
- With keyword-based error detection (for the preservation guarantee),
- when ALL items have error keywords, we detect error_keywords:50 as a
- signal. This makes the data technically crushable.
-
- However, since ALL items are errors, they will ALL be preserved due to
- the preservation guarantee. The end result is the same - no data loss.
- """
- items = [{"id": i, "error": f"Error {i}", "status": "failed"} for i in range(50)]
-
- analysis = analyzer.analyze_array(items)
- assert analysis.crushability is not None
-
- # With keyword-based error detection, all 50 items contain error keywords
- # This IS a signal (error_keywords:50), making the data crushable.
- # However, all 50 items will be preserved due to the preservation guarantee.
- assert analysis.crushability.crushable
- assert "error_keywords:50" in analysis.crushability.signals_present
-
-
-class TestCrushabilityConfidence:
- """Test confidence scoring in crushability analysis."""
-
- @pytest.fixture
- def analyzer(self):
- return SmartAnalyzer(SmartCrusherConfig())
-
- def test_high_confidence_for_clear_cases(self, analyzer):
- """Clear-cut cases should have high confidence."""
- # Low uniqueness - clearly safe
- items = [{"status": "ok", "code": 200} for _ in range(100)]
- analysis = analyzer.analyze_array(items)
- assert analysis.crushability is not None
- assert analysis.crushability.confidence >= 0.8
-
- def test_lower_confidence_for_ambiguous_cases(self, analyzer):
- """Ambiguous cases should have lower confidence."""
- # Medium uniqueness with weak signal
- items = [
- {"id": i, "value": i % 10, "status": "active" if i % 2 == 0 else "inactive"}
- for i in range(100)
- ]
- # Add one error to provide weak signal
- items[50]["error"] = "minor issue"
-
- analysis = analyzer.analyze_array(items)
- assert analysis.crushability is not None
- # Should be lower confidence due to ambiguity
- assert analysis.crushability.confidence <= 0.7
diff --git a/tests/test_relevance.py b/tests/test_relevance.py
index bdfa4de1a..6da26e1a0 100644
--- a/tests/test_relevance.py
+++ b/tests/test_relevance.py
@@ -273,33 +273,6 @@ class TestEmbeddingAvailable:
class TestSmartCrusherIntegration:
"""Integration tests for SmartCrusher with RelevanceScorer."""
- def test_crusher_uses_scorer(self):
- """SmartCrusher uses relevance scorer for context matching."""
- from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig
-
- config = SmartCrusherConfig(
- min_items_to_analyze=3,
- min_tokens_to_crush=0,
- max_items_after_crush=5,
- )
- crusher = SmartCrusher(config=config)
-
- # Verify scorer is initialized
- assert hasattr(crusher, "_scorer")
- assert hasattr(crusher, "_relevance_threshold")
-
- def test_custom_scorer_injection(self):
- """Custom scorer can be injected."""
- from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig
-
- custom_scorer = BM25Scorer(k1=2.0)
- crusher = SmartCrusher(
- config=SmartCrusherConfig(),
- scorer=custom_scorer,
- )
-
- assert crusher._scorer is custom_scorer
-
def test_context_extraction(self):
"""Context is extracted from messages."""
from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig
diff --git a/tests/test_relevance_extra.py b/tests/test_relevance_extra.py
index 62a8fa895..90b6d34e0 100644
--- a/tests/test_relevance_extra.py
+++ b/tests/test_relevance_extra.py
@@ -90,7 +90,7 @@ def test_embedding_numpy_and_model_error_paths(monkeypatch) -> None:
assert embedding._cosine_similarity([1, 0], [0, 1]) == 0.0
monkeypatch.setattr(EmbeddingScorer, "is_available", classmethod(lambda cls: False))
- with pytest.raises(RuntimeError, match="requires sentence-transformers"):
+ with pytest.raises(RuntimeError, match="requires fastembed"):
EmbeddingScorer()._get_model()
diff --git a/tests/test_toin_field_learning.py b/tests/test_toin_field_learning.py
deleted file mode 100644
index ae875cd43..000000000
--- a/tests/test_toin_field_learning.py
+++ /dev/null
@@ -1,737 +0,0 @@
-"""Tests for TOIN Field-Level Learning.
-
-These tests PROVE that field-level learning actually works:
-1. FieldSemantics correctly infers types from retrieval patterns
-2. TOIN populates field_semantics from retrievals
-3. SmartCrusher uses learned semantics to detect important items
-4. End-to-end: important items are preserved based on learned behavior
-
-No hardcoded patterns - all learning is behavior-based.
-"""
-
-import hashlib
-
-import pytest
-
-from headroom.telemetry import (
- ToolIntelligenceNetwork,
- ToolPattern,
- ToolSignature,
- reset_toin,
-)
-from headroom.telemetry.models import FieldSemantics
-
-
[email protected](autouse=True)
-def reset_globals():
- """Reset global state before each test."""
- reset_toin()
- yield
- reset_toin()
-
-
-def _hash_value(value) -> str:
- """Hash a value the same way TOIN does."""
- value_str = str(value)
- return hashlib.sha256(value_str.encode()).hexdigest()[:8]
-
-
-def _hash_field(field_name: str) -> str:
- """Hash a field name the same way TOIN does."""
- return hashlib.sha256(field_name.encode()).hexdigest()[:8]
-
-
-class TestFieldSemanticsLearning:
- """Test that FieldSemantics correctly learns from retrieval patterns."""
-
- def test_identifier_type_inference(self):
- """PROVES: Field used with exact-match queries + high uniqueness = identifier."""
- fs = FieldSemantics(field_hash="test123")
-
- # Simulate: user retrieves items by unique IDs (exact match queries)
- # Each ID is different - high uniqueness
- for i in range(10):
- fs.record_retrieval_value(_hash_value(f"id_{i}"), operator="=")
-
- # Simulate compression stats: all values are unique
- fs.record_compression_stats(
- unique_values=100,
- total_values=100, # uniqueness ratio = 1.0
- most_common_value_hash=_hash_value("id_0"),
- most_common_frequency=0.01, # No dominant value
- )
- fs.record_compression_stats(
- unique_values=100,
- total_values=100,
- most_common_value_hash=_hash_value("id_1"),
- most_common_frequency=0.01,
- )
-
- # Now infer the type
- fs.infer_type()
-
- # VERIFY: Should be classified as identifier
- assert fs.inferred_type == "identifier", (
- f"Expected 'identifier' but got '{fs.inferred_type}'. "
- "High uniqueness + exact match queries should = identifier"
- )
- assert fs.confidence > 0.5, "Should have reasonable confidence"
-
- def test_error_indicator_type_inference(self):
- """PROVES: Field with dominant default + retrievals for non-default = error_indicator."""
- fs = FieldSemantics(field_hash="status_field")
-
- # Simulate: most items have status="success" (the default)
- # But user only retrieves items with status="error" or "failed"
- error_hash = _hash_value("error")
- failed_hash = _hash_value("failed")
- success_hash = _hash_value("success")
-
- # User retrieves "error" and "failed" values (non-default)
- for _ in range(5):
- fs.record_retrieval_value(error_hash, operator="=")
- fs.record_retrieval_value(failed_hash, operator="=")
-
- # Compression stats: 90% have "success" (the default)
- fs.record_compression_stats(
- unique_values=3, # "success", "error", "failed"
- total_values=100,
- most_common_value_hash=success_hash,
- most_common_frequency=0.9, # 90% are "success"
- )
- fs.record_compression_stats(
- unique_values=3,
- total_values=100,
- most_common_value_hash=success_hash,
- most_common_frequency=0.9,
- )
-
- fs.infer_type()
-
- # VERIFY: Should be error_indicator
- assert fs.inferred_type == "error_indicator", (
- f"Expected 'error_indicator' but got '{fs.inferred_type}'. "
- "Dominant default + retrieval of non-default = error indicator"
- )
- assert fs.default_value_hash == success_hash, "Default should be 'success'"
- assert fs.confidence > 0.5, "Should have reasonable confidence"
-
- def test_status_type_inference(self):
- """PROVES: Low cardinality + specific values retrieved = status."""
- fs = FieldSemantics(field_hash="state_field")
-
- # Simulate: field has few unique values (low cardinality)
- # User retrieves the same few values repeatedly
- pending_hash = _hash_value("pending")
- active_hash = _hash_value("active")
-
- for _ in range(6):
- fs.record_retrieval_value(pending_hash, operator="=")
- for _ in range(4):
- fs.record_retrieval_value(active_hash, operator="=")
-
- # Compression stats: only 5 unique values across 100 items
- fs.record_compression_stats(
- unique_values=5,
- total_values=100, # uniqueness ratio = 0.05 (very low)
- most_common_value_hash=None,
- most_common_frequency=0.3, # No overwhelming default
- )
- fs.record_compression_stats(
- unique_values=5,
- total_values=100,
- most_common_value_hash=None,
- most_common_frequency=0.3,
- )
-
- fs.infer_type()
-
- # VERIFY: Should be status
- assert fs.inferred_type == "status", (
- f"Expected 'status' but got '{fs.inferred_type}'. "
- "Low cardinality + specific values retrieved = status"
- )
-
- def test_score_type_inference(self):
- """PROVES: Range queries = score type."""
- fs = FieldSemantics(field_hash="relevance_field")
-
- # Simulate: user queries with range operators (top-N behavior)
- for _ in range(8):
- fs.record_retrieval_value(_hash_value("0.95"), operator=">")
- for _ in range(4):
- fs.record_retrieval_value(_hash_value("0.90"), operator=">=")
-
- # Compression stats
- fs.record_compression_stats(
- unique_values=50,
- total_values=100,
- most_common_value_hash=None,
- most_common_frequency=0.1,
- )
- fs.record_compression_stats(
- unique_values=50,
- total_values=100,
- most_common_value_hash=None,
- most_common_frequency=0.1,
- )
-
- fs.infer_type()
-
- # VERIFY: Should be score
- assert fs.inferred_type == "score", (
- f"Expected 'score' but got '{fs.inferred_type}'. "
- "Range queries (>, >=) should = score type"
- )
-
- def test_content_type_inference(self):
- """PROVES: Contains/text search queries = content type."""
- fs = FieldSemantics(field_hash="description_field")
-
- # Simulate: user does text search on this field
- for i in range(10):
- fs.record_retrieval_value(_hash_value(f"search_term_{i}"), operator="contains")
-
- # Compression stats: high uniqueness (different descriptions)
- fs.record_compression_stats(
- unique_values=90,
- total_values=100,
- most_common_value_hash=None,
- most_common_frequency=0.05,
- )
- fs.record_compression_stats(
- unique_values=90,
- total_values=100,
- most_common_value_hash=None,
- most_common_frequency=0.05,
- )
-
- fs.infer_type()
-
- # VERIFY: Should be content
- assert fs.inferred_type == "content", (
- f"Expected 'content' but got '{fs.inferred_type}'. "
- "Contains queries should = content type"
- )
-
- def test_is_value_important_for_error_indicator(self):
- """PROVES: For error_indicator, non-default values are important."""
- fs = FieldSemantics(field_hash="status")
-
- error_hash = _hash_value("error")
- success_hash = _hash_value("success")
-
- # Set up as error_indicator
- fs.inferred_type = "error_indicator"
- fs.confidence = 0.8
- fs.default_value_hash = success_hash
- fs.important_value_hashes = [error_hash]
-
- # VERIFY
- assert fs.is_value_important(error_hash), "Error value should be important"
- assert not fs.is_value_important(success_hash), "Default value should NOT be important"
-
- def test_is_value_important_for_status(self):
- """PROVES: For status fields, retrieved values are important."""
- fs = FieldSemantics(field_hash="state")
-
- pending_hash = _hash_value("pending")
- unknown_hash = _hash_value("never_retrieved")
-
- # Set up as status
- fs.inferred_type = "status"
- fs.confidence = 0.7
- fs.value_retrieval_frequency = {pending_hash: 5}
-
- # VERIFY
- assert fs.is_value_important(pending_hash), "Retrieved value should be important"
- assert not fs.is_value_important(unknown_hash), (
- "Never-retrieved value should NOT be important"
- )
-
-
-class TestTOINFieldLearningIntegration:
- """Test that TOIN correctly integrates field-level learning."""
-
- def test_record_retrieval_populates_field_semantics(self):
- """PROVES: record_retrieval with items actually populates field_semantics."""
- toin = ToolIntelligenceNetwork()
-
- # Create a tool signature
- items = [
- {"id": "123", "status": "ok", "value": 100},
- {"id": "456", "status": "error", "value": 200},
- ]
- sig = ToolSignature.from_items(items)
-
- # Record retrieval with items - THIS IS WHERE LEARNING HAPPENS
- toin.record_retrieval(
- tool_signature_hash=sig.structure_hash,
- retrieval_type="full",
- query="status=error",
- query_fields=["status"],
- retrieved_items=items,
- )
-
- # VERIFY: pattern should have field_semantics populated
- pattern = toin._patterns.get(sig.structure_hash)
- assert pattern is not None, "Pattern should exist"
- assert len(pattern.field_semantics) > 0, (
- f"field_semantics should be populated after retrieval. Got: {pattern.field_semantics}"
- )
-
- # Check that field hashes match expected fields
- expected_field_hashes = {_hash_field(f) for f in ["id", "status", "value"]}
- actual_field_hashes = set(pattern.field_semantics.keys())
- assert expected_field_hashes == actual_field_hashes, (
- f"Expected field hashes {expected_field_hashes}, got {actual_field_hashes}"
- )
-
- def test_repeated_retrievals_trigger_type_inference(self):
- """PROVES: After multiple retrievals, TOIN infers field types."""
- toin = ToolIntelligenceNetwork()
-
- # Simulate items with status field
- items = [
- {"status": "success"},
- {"status": "success"},
- {"status": "success"},
- {"status": "error"},
- ]
- sig = ToolSignature.from_items(items)
-
- # Simulate 6 retrievals (enough to trigger inference at retrieval 5)
- for _ in range(6):
- # User always retrieves items with error status
- toin.record_retrieval(
- tool_signature_hash=sig.structure_hash,
- retrieval_type="full",
- query="status=error",
- query_fields=["status"],
- retrieved_items=[{"status": "error"}],
- )
- # Also record compression to get enough data for inference
- toin.record_compression(
- tool_signature=sig,
- original_count=100,
- compressed_count=10,
- original_tokens=1000,
- compressed_tokens=100,
- strategy="top_n",
- items=items,
- )
-
- # VERIFY: After enough retrievals, type should be inferred
- pattern = toin._patterns.get(sig.structure_hash)
- status_hash = _hash_field("status")
- assert status_hash in pattern.field_semantics, "Status field should be tracked"
-
- status_sem = pattern.field_semantics[status_hash]
- # The type should be inferred (not unknown) after enough data
- assert status_sem.retrieval_count >= 6, (
- f"Should have 6+ retrievals, got {status_sem.retrieval_count}"
- )
-
- def test_get_recommendation_includes_field_semantics(self):
- """PROVES: get_recommendation returns learned field_semantics."""
- toin = ToolIntelligenceNetwork()
-
- # Set up pattern with learned field_semantics
- items = [{"id": "123", "status": "ok"}]
- sig = ToolSignature.from_items(items)
-
- # Record enough data
- for i in range(10):
- toin.record_retrieval(
- tool_signature_hash=sig.structure_hash,
- retrieval_type="full",
- query=f"id={i}",
- query_fields=["id"],
- retrieved_items=[{"id": str(i), "status": "ok"}],
- )
- toin.record_compression(
- tool_signature=sig,
- original_count=100,
- compressed_count=10,
- original_tokens=1000,
- compressed_tokens=100,
- strategy="top_n",
- items=items,
- )
-
- # Get recommendation
- hint = toin.get_recommendation(sig)
-
- # VERIFY: hint should include field_semantics
- assert hint is not None, "Should get a recommendation"
- # field_semantics might be empty if confidence is too low,
- # but the attribute should exist
- assert hasattr(hint, "field_semantics"), "Hint should have field_semantics attribute"
-
- def test_field_semantics_persisted_correctly(self):
- """PROVES: field_semantics survives to_dict/from_dict round-trip."""
- # ToolPattern is already imported at module level from headroom.telemetry
-
- # Create pattern with field_semantics
- pattern = ToolPattern(tool_signature_hash="test123")
-
- # Add field semantics
- fs = FieldSemantics(field_hash="field123")
- fs.inferred_type = "error_indicator"
- fs.confidence = 0.8
- fs.important_value_hashes = ["value1", "value2"]
- fs.default_value_hash = "default"
- pattern.field_semantics["field123"] = fs
-
- # Round-trip through dict
- d = pattern.to_dict()
- pattern2 = ToolPattern.from_dict(d)
-
- # VERIFY: field_semantics preserved
- assert "field123" in pattern2.field_semantics, "field_semantics should be preserved"
- fs2 = pattern2.field_semantics["field123"]
- assert fs2.inferred_type == "error_indicator"
- assert fs2.confidence == 0.8
- assert fs2.important_value_hashes == ["value1", "value2"]
-
-
-class TestSmartCrusherUsesLearnedSemantics:
- """Test that SmartCrusher actually uses learned field semantics."""
-
- def test_detect_items_by_learned_semantics_finds_important_items(self):
- """PROVES: _detect_items_by_learned_semantics correctly identifies items."""
- from headroom.transforms.smart_crusher import _detect_items_by_learned_semantics
-
- # Create field semantics that knows "error" is important
- status_hash = _hash_field("status")
- error_hash = _hash_value("error")
- success_hash = _hash_value("success")
-
- fs = FieldSemantics(field_hash=status_hash)
- fs.inferred_type = "error_indicator"
- fs.confidence = 0.8
- fs.default_value_hash = success_hash
- fs.important_value_hashes = [error_hash]
- fs.value_retrieval_frequency = {error_hash: 10}
-
- field_semantics = {status_hash: fs}
-
- # Test items - index 1 has error status
- items = [
- {"status": "success", "message": "all good"},
- {"status": "error", "message": "something failed"}, # <-- This should be detected
- {"status": "success", "message": "also good"},
- ]
-
- # VERIFY
- important_indices = _detect_items_by_learned_semantics(items, field_semantics)
- assert 1 in important_indices, (
- f"Index 1 (error status) should be detected as important. "
- f"Got indices: {important_indices}"
- )
- assert 0 not in important_indices, "Index 0 (success) should not be important"
- assert 2 not in important_indices, "Index 2 (success) should not be important"
-
- def test_detect_items_handles_empty_semantics(self):
- """PROVES: Function handles edge cases gracefully."""
- from headroom.transforms.smart_crusher import _detect_items_by_learned_semantics
-
- items = [{"status": "ok"}]
-
- # Empty semantics
- assert _detect_items_by_learned_semantics(items, {}) == []
- assert _detect_items_by_learned_semantics(items, None) == []
- assert _detect_items_by_learned_semantics([], {"x": FieldSemantics(field_hash="x")}) == []
-
- def test_detect_items_requires_confidence(self):
- """PROVES: Low confidence semantics are ignored."""
- from headroom.transforms.smart_crusher import _detect_items_by_learned_semantics
-
- status_hash = _hash_field("status")
- error_hash = _hash_value("error")
-
- fs = FieldSemantics(field_hash=status_hash)
- fs.inferred_type = "error_indicator"
- fs.confidence = 0.1 # TOO LOW
- fs.important_value_hashes = [error_hash]
-
- items = [{"status": "error"}]
-
- # VERIFY: Low confidence = ignored
- result = _detect_items_by_learned_semantics(items, {status_hash: fs})
- assert result == [], "Low confidence semantics should be ignored"
-
-
-class TestEndToEndFieldLearning:
- """End-to-end tests proving the full learning pipeline works."""
-
- def test_full_pipeline_learns_and_applies(self):
- """PROVES: End-to-end learning from retrieval to compression."""
- toin = ToolIntelligenceNetwork()
-
- # PHASE 1: Learning - User retrieves items with error status
- # Simulating: "show me all failed items"
- items_with_errors = [
- {"id": "1", "status": "success", "data": "..."},
- {"id": "2", "status": "error", "data": "..."}, # Retrieved
- {"id": "3", "status": "success", "data": "..."},
- {"id": "4", "status": "failed", "data": "..."}, # Retrieved
- ]
- sig = ToolSignature.from_items(items_with_errors)
-
- # User keeps retrieving error/failed items (learning behavior)
- for _ in range(5):
- toin.record_retrieval(
- tool_signature_hash=sig.structure_hash,
- retrieval_type="full",
- query="status!=success",
- query_fields=["status"],
- retrieved_items=[
- {"id": "2", "status": "error", "data": "..."},
- {"id": "4", "status": "failed", "data": "..."},
- ],
- )
- toin.record_compression(
- tool_signature=sig,
- original_count=100,
- compressed_count=10,
- original_tokens=1000,
- compressed_tokens=100,
- strategy="top_n",
- items=[
- {"id": str(i), "status": "success" if i % 5 != 0 else "error", "data": "..."}
- for i in range(100)
- ],
- )
-
- # PHASE 2: Verify learning occurred
- pattern = toin._patterns.get(sig.structure_hash)
- assert pattern is not None
- assert len(pattern.field_semantics) > 0, "Should have learned field semantics"
-
- # Check status field was learned
- status_hash = _hash_field("status")
- if status_hash in pattern.field_semantics:
- status_sem = pattern.field_semantics[status_hash]
- # Error value should be tracked
- error_hash = _hash_value("error")
- failed_hash = _hash_value("failed")
- assert (
- error_hash in status_sem.important_value_hashes
- or failed_hash in status_sem.important_value_hashes
- ), "Error/failed values should be marked as important"
-
- def test_recommendation_hint_includes_learned_semantics(self):
- """PROVES: TOIN recommendation includes learned field semantics for SmartCrusher."""
- toin = ToolIntelligenceNetwork()
-
- # Set up sufficient learning
- items = [{"status": "ok", "id": "123"}]
- sig = ToolSignature.from_items(items)
-
- # Create pattern with confident field semantics by directly manipulating internal state
- # (This is a test - in real code, patterns are created via record_* methods)
- pattern = ToolPattern(tool_signature_hash=sig.structure_hash)
- toin._patterns[sig.structure_hash] = pattern
-
- status_hash = _hash_field("status")
- fs = FieldSemantics(field_hash=status_hash)
- fs.inferred_type = "error_indicator"
- fs.confidence = 0.8 # High confidence
- fs.retrieval_count = 10
- fs.important_value_hashes = [_hash_value("error")]
- pattern.field_semantics[status_hash] = fs
-
- # Ensure pattern has enough data for recommendation
- pattern.total_compressions = 10
- pattern.sample_size = 10 # Required by min_samples_for_recommendation
- pattern.confidence = 0.5
-
- # Get recommendation
- hint = toin.get_recommendation(sig)
-
- # VERIFY
- assert hint is not None
- assert len(hint.field_semantics) > 0, (
- f"Recommendation should include field_semantics. Got: {hint.field_semantics}"
- )
- assert status_hash in hint.field_semantics
-
-
-class TestFieldSemanticsMemoryBounds:
- """Test that memory bounds are enforced."""
-
- def test_important_values_bounded(self):
- """PROVES: important_value_hashes stays within bounds."""
- fs = FieldSemantics(field_hash="test")
-
- # Add more values than MAX_IMPORTANT_VALUES
- for i in range(fs.MAX_IMPORTANT_VALUES + 20):
- fs.record_retrieval_value(_hash_value(f"value_{i}"))
-
- # VERIFY: bounded
- assert len(fs.important_value_hashes) <= fs.MAX_IMPORTANT_VALUES
-
- def test_value_frequency_bounded(self):
- """PROVES: value_retrieval_frequency stays within bounds."""
- fs = FieldSemantics(field_hash="test")
-
- # Add more values than MAX_VALUE_FREQUENCY_ENTRIES
- for i in range(fs.MAX_VALUE_FREQUENCY_ENTRIES + 20):
- fs.record_retrieval_value(_hash_value(f"value_{i}"))
-
- # VERIFY: bounded
- assert len(fs.value_retrieval_frequency) <= fs.MAX_VALUE_FREQUENCY_ENTRIES
-
-
-class TestProductionCodePath:
- """Integration tests for the ACTUAL production code path.
-
- These tests verify that CompressionStore -> TOIN integration works,
- not just TOIN in isolation. This is critical because the unit tests
- can pass while the production integration is broken.
- """
-
- def test_compression_store_passes_items_to_toin(self):
- """PROVES: CompressionStore.process_pending_feedback passes retrieved_items to TOIN.
-
- This is the integration test that would have caught the original bug
- where retrieved_items was never passed to TOIN in production.
- """
- import json
-
- from headroom.cache.compression_store import CompressionStore
- from headroom.telemetry.toin import get_toin, reset_toin
-
- reset_toin()
- toin = get_toin()
-
- # Create a store with feedback enabled
- store = CompressionStore(max_entries=100, default_ttl=300, enable_feedback=True)
-
- # Store some compressed content with items that have distinct field values
- items = [
- {"id": "123", "status": "success", "value": 100},
- {"id": "456", "status": "error", "value": 200},
- {"id": "789", "status": "success", "value": 300},
- ]
- compressed_json = json.dumps(items)
- original_json = json.dumps(items * 10) # Original was bigger
-
- # Create a tool signature hash for this structure
- sig = ToolSignature.from_items(items)
-
- hash_key = store.store(
- original=original_json,
- compressed=compressed_json,
- original_tokens=1000,
- compressed_tokens=100,
- original_item_count=30,
- compressed_item_count=3,
- tool_name="test_api",
- tool_call_id="call_123",
- tool_signature_hash=sig.structure_hash,
- compression_strategy="top_n",
- )
-
- # Simulate a retrieval (this triggers the feedback loop)
- store.retrieve(hash_key, query="status=error")
-
- # Process pending feedback - THIS IS WHERE THE BUG WAS
- store.process_pending_feedback()
-
- # VERIFY: TOIN should have received the items and learned from them
- pattern = toin._patterns.get(sig.structure_hash)
- assert pattern is not None, "TOIN should have a pattern for this tool"
-
- # The key assertion: field_semantics should be populated
- # This would have FAILED before the fix because retrieved_items wasn't passed
- assert len(pattern.field_semantics) > 0, (
- "TOIN should have learned field semantics from the retrieved items. "
- "If this fails, CompressionStore is not passing retrieved_items to TOIN."
- )
-
- # Verify specific fields were learned
- id_hash = _hash_field("id")
- status_hash = _hash_field("status")
- value_hash = _hash_field("value")
-
- learned_fields = set(pattern.field_semantics.keys())
- expected_fields = {id_hash, status_hash, value_hash}
- assert expected_fields == learned_fields, (
- f"Expected fields {expected_fields}, got {learned_fields}"
- )
-
- def test_compression_store_handles_wrapped_arrays(self):
- """PROVES: CompressionStore correctly extracts items from wrapped arrays."""
- import json
-
- from headroom.cache.compression_store import CompressionStore
- from headroom.telemetry.toin import get_toin, reset_toin
-
- reset_toin()
- toin = get_toin()
-
- store = CompressionStore(max_entries=100, default_ttl=300, enable_feedback=True)
-
- # Content wrapped in {"results": [...]} pattern
- items = [{"name": "test", "score": 0.95}]
- wrapped_content = json.dumps({"results": items, "total": 1})
-
- sig = ToolSignature.from_items(items)
-
- hash_key = store.store(
- original=wrapped_content,
- compressed=wrapped_content,
- original_tokens=100,
- compressed_tokens=100,
- original_item_count=1,
- compressed_item_count=1,
- tool_name="search_api",
- tool_call_id="call_456",
- tool_signature_hash=sig.structure_hash,
- compression_strategy="top_n",
- )
-
- store.retrieve(hash_key)
- store.process_pending_feedback()
-
- # VERIFY: Items were extracted from wrapped structure
- pattern = toin._patterns.get(sig.structure_hash)
- assert pattern is not None
- assert len(pattern.field_semantics) > 0, (
- "TOIN should extract items from wrapped arrays like {'results': [...]}"
- )
-
- def test_compression_store_handles_invalid_json(self):
- """PROVES: CompressionStore gracefully handles invalid JSON."""
- from headroom.cache.compression_store import CompressionStore
- from headroom.telemetry.toin import get_toin, reset_toin
-
- reset_toin()
- get_toin() # Initialize TOIN for feedback loop
-
- store = CompressionStore(max_entries=100, default_ttl=300, enable_feedback=True)
-
- # Store invalid JSON content
- invalid_json = "not valid json {"
-
- hash_key = store.store(
- original=invalid_json,
- compressed=invalid_json,
- original_tokens=10,
- compressed_tokens=10,
- original_item_count=0,
- compressed_item_count=0,
- tool_name="broken_api",
- tool_call_id="call_789",
- tool_signature_hash="invalid123",
- compression_strategy="none",
- )
-
- # This should not crash
- store.retrieve(hash_key)
- store.process_pending_feedback()
-
- # VERIFY: No crash, pattern may or may not exist but no exception
- # The main assertion is that we got here without exception
diff --git a/tests/test_toin_integration.py b/tests/test_toin_integration.py
index 4770aa87f..860228781 100644
--- a/tests/test_toin_integration.py
+++ b/tests/test_toin_integration.py
@@ -8,7 +8,6 @@ Tests the complete flow:
5. Future compressions get improved recommendations
"""
-import json
import tempfile
from pathlib import Path
@@ -18,18 +17,11 @@ from headroom.cache.compression_store import (
get_compression_store,
reset_compression_store,
)
-from headroom.config import CCRConfig
-from headroom.telemetry import ToolSignature
from headroom.telemetry.toin import (
TOINConfig,
- ToolIntelligenceNetwork,
get_toin,
reset_toin,
)
-from headroom.transforms.smart_crusher import (
- SmartCrusher,
- SmartCrusherConfig,
-)
@pytest.fixture
@@ -55,332 +47,3 @@ def fresh_store():
store = get_compression_store(max_entries=100, default_ttl=300)
yield store
reset_compression_store()
-
-
-class TestTOINIntegration:
- """Test the full TOIN feedback loop."""
-
- def test_compression_records_correct_hash(self, fresh_toin, fresh_store):
- """Test that SmartCrusher records the correct tool_signature_hash in store."""
- # Create test data
- items = [{"id": i, "score": 100 - i, "name": f"Item {i}"} for i in range(50)]
- content = json.dumps(items)
-
- # Create a SmartCrusher with CCR enabled
- ccr_config = CCRConfig(enabled=True, inject_retrieval_marker=False)
- crusher = SmartCrusher(
- SmartCrusherConfig(max_items_after_crush=10),
- ccr_config=ccr_config,
- )
-
- # Compress the content
- crushed, was_modified, info = crusher._smart_crush_content(content, tool_name="test_tool")
-
- assert was_modified, "Content should be modified by compression"
-
- # Verify the store has an entry with the correct tool_signature_hash
- stats = fresh_store.get_stats()
- assert stats["entry_count"] >= 1, "Store should have at least one entry"
-
- # Get the entry and verify it has tool_signature_hash
- # We need to find the hash key from the store
- entries = [entry for _, entry in fresh_store._backend.items()]
- assert len(entries) >= 1, "Should have at least one entry"
-
- entry = entries[0]
- assert entry.tool_signature_hash is not None, "Entry should have tool_signature_hash"
- assert entry.compression_strategy is not None, "Entry should have compression_strategy"
-
- # Verify the hash matches what ToolSignature would generate
- expected_signature = ToolSignature.from_items(items)
- assert entry.tool_signature_hash == expected_signature.structure_hash, (
- "Stored hash should match ToolSignature.structure_hash"
- )
-
- def test_retrieval_updates_toin_strategy_success(self, fresh_toin, fresh_store):
- """Test that retrieval events update TOIN strategy success rates."""
- # Create test data
- items = [{"id": i, "score": 100 - i, "name": f"Item {i}"} for i in range(50)]
- signature = ToolSignature.from_items(items)
-
- # Record some compressions with fresh_toin
- for _ in range(5):
- fresh_toin.record_compression(
- tool_signature=signature,
- original_count=50,
- compressed_count=10,
- original_tokens=5000,
- compressed_tokens=1000,
- strategy="smart_sample",
- )
-
- # Get initial strategy success rate
- pattern = fresh_toin._patterns.get(signature.structure_hash)
- assert pattern is not None, "Pattern should exist after compressions"
-
- initial_rate = pattern.strategy_success_rates.get("smart_sample", 1.0)
- assert initial_rate > 0, "Initial success rate should be positive"
-
- # Simulate retrieval events (which indicate compression was too aggressive)
- for _ in range(3):
- fresh_toin.record_retrieval(
- tool_signature_hash=signature.structure_hash,
- retrieval_type="full",
- query="test query",
- query_fields=["id"],
- strategy="smart_sample",
- )
-
- # Verify success rate decreased
- final_rate = pattern.strategy_success_rates.get("smart_sample", 1.0)
- assert final_rate < initial_rate, (
- f"Success rate should decrease after retrievals: {initial_rate} -> {final_rate}"
- )
-
- def test_full_feedback_loop(self, fresh_toin, fresh_store):
- """Test the complete feedback loop: compress → retrieve → learn → recommend."""
- # Create test data
- items = [{"id": i, "score": 100 - i, "name": f"Item {i}"} for i in range(50)]
- content = json.dumps(items)
- signature = ToolSignature.from_items(items)
-
- # Step 1: Compress with SmartCrusher (records to TOIN)
- ccr_config = CCRConfig(enabled=True, inject_retrieval_marker=False)
- crusher = SmartCrusher(
- SmartCrusherConfig(max_items_after_crush=10, use_feedback_hints=True),
- ccr_config=ccr_config,
- )
-
- # Multiple compressions to build pattern
- for _ in range(5):
- crusher._smart_crush_content(content, tool_name="test_tool")
-
- # Step 2: Verify TOIN has a pattern
- pattern = fresh_toin._patterns.get(signature.structure_hash)
- assert pattern is not None, "TOIN should have a pattern after compressions"
- assert pattern.total_compressions >= 5, "Should have recorded 5 compressions"
-
- # Step 3: Simulate retrievals (indicating compression was too aggressive)
- # Find the stored entry hash
- entries = [entry for _, entry in fresh_store._backend.items()]
- assert len(entries) > 0, "Should have cached entries"
-
- # Retrieve multiple times to trigger learning
- for entry in entries[:3]:
- fresh_store.retrieve(entry.hash, query="find all items")
-
- # Step 4: Verify TOIN learned from retrievals
- pattern = fresh_toin._patterns.get(signature.structure_hash)
- assert pattern is not None, "Pattern should exist after retrievals"
-
- # CRITICAL: Verify field-level learning actually happened
- # This assertion would have caught the bug where compression_store
- # wasn't passing retrieved_items to TOIN
- assert len(pattern.field_semantics) > 0, (
- "TOIN should learn field semantics from retrieved items. "
- "If this fails, the production code path (CompressionStore -> TOIN) is broken."
- )
-
- # Verify retrieval stats were updated
- assert pattern.total_retrievals >= 1, "Should have recorded retrievals"
-
- recommendation = fresh_toin.get_recommendation(signature, "find all items")
-
- # After many retrievals, TOIN should recommend more items
- # Default is 15-20, but with high retrieval rate it should go higher
- assert recommendation.confidence > 0, "Recommendation should have confidence"
-
- def test_preserve_fields_used_in_compression(self, fresh_toin, fresh_store):
- """Test that TOIN preserve_fields are used during compression planning."""
- # Create test data with specific fields
- items = [{"id": i, "score": 100 - i, "category": f"cat_{i % 3}"} for i in range(50)]
- signature = ToolSignature.from_items(items)
-
- # Record compressions and retrievals that query the "category" field
- for _ in range(10):
- fresh_toin.record_compression(
- tool_signature=signature,
- original_count=50,
- compressed_count=10,
- original_tokens=5000,
- compressed_tokens=1000,
- strategy="smart_sample",
- )
-
- # Record retrievals that query "category"
- for _ in range(5):
- fresh_toin.record_retrieval(
- tool_signature_hash=signature.structure_hash,
- retrieval_type="search",
- query="category:cat_1",
- query_fields=["category"],
- strategy="smart_sample",
- )
-
- # Get recommendation - should now preserve "category" field
- recommendation = fresh_toin.get_recommendation(signature, "find all items in cat_1")
-
- # Verify the recommendation reflects learning
- assert recommendation.source in ("local", "network", "default"), (
- f"Should have a valid source: {recommendation.source}"
- )
-
- def test_instance_id_stable_across_restarts(self):
- """Test that instance_id is stable across restarts."""
- reset_toin()
- with tempfile.TemporaryDirectory() as tmpdir:
- storage_path = str(Path(tmpdir) / "toin_stable.json")
-
- # Create first TOIN instance
- toin1 = ToolIntelligenceNetwork(TOINConfig(storage_path=storage_path))
- instance_id_1 = toin1._instance_id
-
- # Create second instance with same path (simulating restart)
- toin2 = ToolIntelligenceNetwork(TOINConfig(storage_path=storage_path))
- instance_id_2 = toin2._instance_id
-
- # Instance IDs should be the same since derived from path
- assert instance_id_1 == instance_id_2, (
- f"Instance ID should be stable: {instance_id_1} vs {instance_id_2}"
- )
-
- def test_atomic_save(self):
- """Test that save() uses atomic writes."""
- reset_toin()
- with tempfile.TemporaryDirectory() as tmpdir:
- storage_path = str(Path(tmpdir) / "toin_atomic.json")
- toin = ToolIntelligenceNetwork(TOINConfig(storage_path=storage_path))
-
- # Create some data
- items = [{"id": i, "name": f"test_{i}"} for i in range(10)]
- signature = ToolSignature.from_items(items)
-
- toin.record_compression(
- tool_signature=signature,
- original_count=10,
- compressed_count=5,
- original_tokens=1000,
- compressed_tokens=500,
- strategy="smart_sample",
- )
-
- # Save
- toin.save()
-
- # Verify file exists and is valid JSON
- saved_path = Path(storage_path)
- assert saved_path.exists(), "Save file should exist"
-
- with open(saved_path) as f:
- data = json.load(f)
-
- assert "patterns" in data, "Saved data should have patterns"
- assert len(data["patterns"]) > 0, "Should have at least one pattern"
-
- def test_query_patterns_merged_on_import(self):
- """Test that query patterns are merged when importing patterns."""
- reset_toin()
- with tempfile.TemporaryDirectory() as tmpdir:
- storage_path = str(Path(tmpdir) / "toin_merge.json")
- toin = ToolIntelligenceNetwork(TOINConfig(storage_path=storage_path))
-
- # Create local pattern
- items = [{"id": i, "name": f"test_{i}"} for i in range(10)]
- signature = ToolSignature.from_items(items)
-
- toin.record_compression(
- tool_signature=signature,
- original_count=10,
- compressed_count=5,
- original_tokens=1000,
- compressed_tokens=500,
- strategy="smart_sample",
- )
-
- # Record local query pattern
- toin.record_retrieval(
- tool_signature_hash=signature.structure_hash,
- retrieval_type="search",
- query="local_pattern:value",
- query_fields=["local_pattern"],
- strategy="smart_sample",
- )
-
- # Create import data with different query pattern
- import_data = {
- "patterns": {
- signature.structure_hash: {
- "tool_signature_hash": signature.structure_hash,
- "total_compressions": 100,
- "total_retrievals": 20,
- "avg_original_count": 50,
- "avg_compressed_count": 10,
- "avg_compression_ratio": 0.2,
- "retrieval_rate": 0.2,
- "common_query_patterns": ["imported_pattern:x"],
- "strategy_success_rates": {"smart_sample": 0.8},
- "preserve_fields": ["imported_field"],
- "user_count": 50,
- "last_updated": 1234567890.0,
- }
- }
- }
-
- # Import patterns
- toin.import_patterns(import_data)
-
- # Verify patterns were merged
- pattern = toin._patterns.get(signature.structure_hash)
- assert pattern is not None, "Pattern should exist after merge"
-
- # Check that both query patterns exist
- assert "imported_pattern:x" in pattern.common_query_patterns, (
- "Imported query pattern should be present"
- )
-
-
-class TestStoreToTOINHash:
- """Test the hash correlation between compression_store and TOIN."""
-
- def test_hash_matches_between_store_and_toin(self, fresh_toin, fresh_store):
- """Test that the hash stored in compression_store matches TOIN events."""
- # Use 50 items with score field to ensure compression threshold is met
- items = [{"id": i, "score": 100 - i, "name": f"Item {i}"} for i in range(50)]
- content = json.dumps(items)
- signature = ToolSignature.from_items(items)
-
- # Compress with aggressive settings to ensure items are actually reduced
- ccr_config = CCRConfig(enabled=True, inject_retrieval_marker=False)
- crusher = SmartCrusher(
- SmartCrusherConfig(max_items_after_crush=10),
- ccr_config=ccr_config,
- )
- crushed, was_modified, info = crusher._smart_crush_content(content, tool_name="hash_test")
-
- # Verify compression actually happened
- assert was_modified, f"Content should be modified by compression: {info}"
-
- # Get the stored hash
- entries = [entry for _, entry in fresh_store._backend.items()]
- assert len(entries) >= 1, (
- f"Should have stored entry. Modified: {was_modified}, Info: {info}"
- )
- stored_hash = entries[0].tool_signature_hash
-
- # Verify it matches ToolSignature
- assert stored_hash == signature.structure_hash, (
- f"Store hash {stored_hash} should match signature {signature.structure_hash}"
- )
-
- # Verify TOIN can receive events for this hash
- fresh_toin.record_compression(
- tool_signature=signature,
- original_count=50,
- compressed_count=10,
- original_tokens=5000,
- compressed_tokens=1000,
- strategy="smart_sample",
- )
-
- pattern = fresh_toin._patterns.get(signature.structure_hash)
- assert pattern is not None, "TOIN should have pattern for same hash"
diff --git a/tests/test_transforms/test_anchor_selector.py b/tests/test_transforms/test_anchor_selector.py
deleted file mode 100644
index aa73c4c70..000000000
--- a/tests/test_transforms/test_anchor_selector.py
+++ /dev/null
@@ -1,1186 +0,0 @@
-"""Tests for AnchorSelector - Adaptive position-based anchor allocation.
-
-Comprehensive tests covering:
-- Adversarial positions: Important data NOT at expected positions
-- Size adaptation: Anchor allocation scaling with array size
-- Pattern-aware anchoring: Different strategies for different data patterns
-- Query-aware anchoring: Query-based anchor adjustment
-- Information density: Unique/informative item selection
-- Coverage metrics: Distribution coverage verification
-- Edge cases: Boundary conditions and special scenarios
-
-These tests verify the AnchorSelector replaces the static "first 3 + last 2"
-preservation with adaptive, pattern-aware anchor selection.
-"""
-
-import json
-
-import pytest
-
-from headroom import OpenAIProvider, SmartCrusherConfig, Tokenizer
-from headroom.transforms.smart_crusher import (
- SmartAnalyzer,
- SmartCrusher,
-)
-
-# =============================================================================
-# Test Fixtures
-# =============================================================================
-
-_provider = OpenAIProvider()
-
-
-def get_tokenizer(model: str = "gpt-4o") -> Tokenizer:
- """Get a tokenizer for tests using OpenAI provider."""
- token_counter = _provider.get_token_counter(model)
- return Tokenizer(token_counter, model)
-
-
-def tokenizer():
- """Provide a tokenizer for tests."""
- return get_tokenizer()
-
-
-def default_config():
- """Default SmartCrusherConfig for testing."""
- return SmartCrusherConfig(
- enabled=True,
- min_items_to_analyze=3,
- min_tokens_to_crush=0, # Always crush for tests
- max_items_after_crush=10,
- variance_threshold=2.0,
- )
-
-
-def analyzer(default_config):
- """SmartAnalyzer instance for testing."""
- return SmartAnalyzer(default_config)
-
-
-def crusher(default_config):
- """SmartCrusher instance for testing."""
- return SmartCrusher(default_config)
-
-
-# =============================================================================
-# Test Data Generators
-# =============================================================================
-
-
-def generate_uniform_items(n: int, value: int = 0) -> list[dict]:
- """Generate array of identical items."""
- return [{"id": "same", "value": value, "status": "ok"} for _ in range(n)]
-
-
-def generate_numbered_items(n: int, with_value: bool = True) -> list[dict]:
- """Generate items with sequential IDs for position tracking."""
- items = []
- for i in range(n):
- item = {"id": i, "name": f"Item {i}"}
- if with_value:
- item["value"] = i * 10
- items.append(item)
- return items
-
-
-def generate_time_series_data(
- n: int = 50,
- spike_positions: list[int] | None = None,
- spike_value: float = 1000.0,
-) -> list[dict]:
- """Generate time series with optional spikes at specific positions."""
- items = []
- for i in range(n):
- value = 100.0 + (i * 0.5) # Slight upward trend
- if spike_positions and i in spike_positions:
- value = spike_value
- items.append(
- {
- "timestamp": f"2025-01-{(i % 28) + 1:02d}T{(i % 24):02d}:00:00Z",
- "value": value,
- "metric": "cpu_usage",
- }
- )
- return items
-
-
-def generate_log_data(
- n: int = 50,
- error_positions: list[int] | None = None,
-) -> list[dict]:
- """Generate log-style data with optional errors at specific positions."""
- levels = ["INFO", "DEBUG", "WARN"]
- items = []
- for i in range(n):
- level = levels[i % len(levels)]
- if error_positions and i in error_positions:
- level = "ERROR"
- message = f"Critical failure at step {i}: connection timeout"
- else:
- message = f"Processing request {i} successfully"
- items.append(
- {
- "level": level,
- "message": message,
- "timestamp": f"2025-01-06T{12 + (i // 60):02d}:{i % 60:02d}:00Z",
- }
- )
- return items
-
-
-def generate_search_results(n: int = 50) -> list[dict]:
- """Generate search results with scores (higher = better, at front)."""
- return [
- {
- "id": f"doc_{i}",
- "title": f"Document {i}",
- "score": 1.0 - (i * 0.02), # Scores decrease as index increases
- "snippet": f"This is a snippet from document {i}...",
- }
- for i in range(n)
- ]
-
-
-def generate_categorized_items(
- categories: dict[str, int],
-) -> list[dict]:
- """Generate items with specific category distribution.
-
- Args:
- categories: Dict mapping category name to count, e.g. {"A": 30, "B": 30, "C": 40}
- """
- items = []
- i = 0
- for category, count in categories.items():
- for _ in range(count):
- items.append({"id": i, "category": category, "name": f"Item {i}"})
- i += 1
- return items
-
-
-# =============================================================================
-# Helper Functions
-# =============================================================================
-
-
-def crush_items(
- items: list[dict],
- tokenizer: Tokenizer,
- max_items: int = 10,
- query: str = "",
- min_items: int = 3,
-) -> list[dict]:
- """Helper to crush items and return the result list.
-
- Args:
- items: Array of items to compress.
- tokenizer: Tokenizer instance.
- max_items: Maximum items after crushing.
- query: Optional query context.
- min_items: Minimum items to analyze.
-
- Returns:
- List of preserved items after crushing.
- """
- messages = [
- {"role": "system", "content": "You are helpful."},
- ]
-
- if query:
- messages.append({"role": "user", "content": query})
- messages.append(
- {
- "role": "assistant",
- "content": None,
- "tool_calls": [
- {
- "id": "call_1",
- "type": "function",
- "function": {"name": "search_items", "arguments": "{}"},
- }
- ],
- }
- )
-
- messages.append({"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)})
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=min_items,
- max_items_after_crush=max_items,
- )
- crusher = SmartCrusher(config)
- result = crusher.apply(messages, tokenizer)
-
- # Parse result
- tool_content = result.messages[-1]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- return json.loads(json_part)
-
-
-def get_preserved_positions(result: list[dict], original_size: int) -> dict[str, list[int]]:
- """Categorize preserved item positions into front/middle/back.
-
- Args:
- result: List of preserved items (must have 'id' field with original index).
- original_size: Size of original array.
-
- Returns:
- Dict with 'front', 'middle', 'back' keys containing lists of preserved indices.
- """
- front_boundary = int(original_size * 0.1)
- back_boundary = int(original_size * 0.9)
-
- positions = {"front": [], "middle": [], "back": []}
-
- for item in result:
- idx = item.get("id")
- if idx is None:
- continue
-
- if isinstance(idx, int):
- if idx < front_boundary:
- positions["front"].append(idx)
- elif idx >= back_boundary:
- positions["back"].append(idx)
- else:
- positions["middle"].append(idx)
-
- return positions
-
-
-# =============================================================================
-# TestAdversarialPositions
-# =============================================================================
-
-
-class TestAdversarialPositions:
- """Test scenarios that break 'first 3 + last 2' assumption.
-
- These tests verify that important data is preserved regardless of position,
- not just because it happens to be at the front or back of the array.
- """
-
- def test_important_data_in_middle(self, tokenizer):
- """Critical error item at position 50 of 100-item array should be preserved.
-
- The error item is in the middle where static anchoring would miss it.
- It should be preserved due to error detection, not position.
- """
- items = [{"id": i, "status": "ok", "value": i * 10} for i in range(100)]
- # Place critical error in the middle
- items[50] = {
- "id": 50,
- "status": "error",
- "error_code": "CRITICAL",
- "message": "Connection failed",
- }
-
- crushed = crush_items(items, tokenizer, max_items=15)
-
- # Error item MUST be preserved regardless of position
- preserved_ids = [item["id"] for item in crushed]
- assert 50 in preserved_ids, "Critical error at position 50 should be preserved"
-
- # Verify it's preserved for the right reason (has error indicator)
- error_items = [item for item in crushed if item.get("status") == "error"]
- assert len(error_items) >= 1, "At least one error item should be preserved"
-
- def test_spike_not_at_boundaries(self, tokenizer):
- """Numeric spike at position 75 of 100-item array must be preserved.
-
- The spike represents an anomaly that should be detected statistically,
- not missed because it's not at the front or back.
- """
- items = generate_time_series_data(100, spike_positions=[75], spike_value=1000.0)
- # Add id field for tracking
- for i, item in enumerate(items):
- item["id"] = i
-
- crushed = crush_items(items, tokenizer, max_items=15)
-
- # Spike MUST be preserved as anomaly
- has_spike = any(item.get("value", 0) > 500 for item in crushed)
- assert has_spike, "Anomalous spike at position 75 should be preserved"
-
- # Verify the specific position was kept
- preserved_ids = [item.get("id") for item in crushed]
- assert 75 in preserved_ids, "Position 75 with spike should be in result"
-
- def test_multiple_spikes_scattered(self, tokenizer):
- """Multiple spikes at positions 25, 50, 75 should all be preserved."""
- spike_positions = [25, 50, 75]
- items = generate_time_series_data(100, spike_positions=spike_positions, spike_value=999.0)
- for i, item in enumerate(items):
- item["id"] = i
-
- crushed = crush_items(items, tokenizer, max_items=15)
-
- # All spikes should be preserved
- preserved_ids = {item.get("id") for item in crushed}
- for pos in spike_positions:
- assert pos in preserved_ids, f"Spike at position {pos} should be preserved"
-
- def test_first_items_identical(self, tokenizer):
- """First 10 identical items - should NOT waste all anchor slots on them.
-
- When first N items are identical, we should keep at most 1-2 of them,
- not waste 3 anchor slots on duplicates.
- """
- # First 10 items are identical
- identical_items = [{"id": "same", "value": 0, "type": "duplicate"} for _ in range(10)]
- # Rest are unique
- unique_items = [{"id": f"unique_{i}", "value": i * 10, "type": "unique"} for i in range(90)]
- items = identical_items + unique_items
-
- crushed = crush_items(items, tokenizer, max_items=10)
-
- # Should NOT have multiple identical items
- same_count = sum(1 for item in crushed if item.get("id") == "same")
- assert same_count <= 2, f"Got {same_count} identical items, expected at most 2"
-
- # Should have some unique items
- unique_count = sum(1 for item in crushed if item.get("type") == "unique")
- assert unique_count >= 5, f"Expected at least 5 unique items, got {unique_count}"
-
- def test_last_items_identical(self, tokenizer):
- """Last 10 identical items - should NOT waste anchor slots.
-
- Similar to above but for back anchors - when last N items are identical,
- we should preserve variety instead of duplicates.
- """
- # First 90 items are unique
- unique_items = [{"id": f"unique_{i}", "value": i * 10, "type": "unique"} for i in range(90)]
- # Last 10 items are identical
- identical_items = [{"id": "same", "value": 100, "type": "duplicate"} for _ in range(10)]
- items = unique_items + identical_items
-
- crushed = crush_items(items, tokenizer, max_items=10)
-
- # Should NOT have multiple identical items
- same_count = sum(1 for item in crushed if item.get("id") == "same")
- assert same_count <= 2, f"Got {same_count} identical items, expected at most 2"
-
- def test_relevant_item_in_middle(self, tokenizer):
- """Item matching query in middle position must be found.
-
- When user queries for a specific item that's at position 42,
- it should be preserved even though it's not at front/back.
- """
- items = [{"id": i, "name": f"item_{i}", "status": "active"} for i in range(100)]
- # Put target item in the middle
- items[42]["name"] = "target_special_item"
- items[42]["description"] = "This is what the user is looking for"
-
- crushed = crush_items(
- items,
- tokenizer,
- max_items=10,
- query="Find target_special_item",
- )
-
- # Query-matched item MUST be preserved
- has_target = any("target_special_item" in item.get("name", "") for item in crushed)
- assert has_target, "Item matching query should be preserved regardless of position"
-
- def test_error_in_middle_of_otherwise_uniform_data(self, tokenizer):
- """Single error in middle of 100 identical OK items must be preserved."""
- items = [{"id": i, "status": "ok", "data": "normal"} for i in range(100)]
- items[47] = {"id": 47, "status": "failed", "error": "Unexpected failure"}
-
- crushed = crush_items(items, tokenizer, max_items=15)
-
- # Error must be preserved
- error_items = [item for item in crushed if item.get("status") == "failed"]
- assert len(error_items) >= 1, "Error item at position 47 should be preserved"
- assert error_items[0]["id"] == 47, "The specific error item should be preserved"
-
-
-# =============================================================================
-# TestSizeAdaptation
-# =============================================================================
-
-
-class TestSizeAdaptation:
- """Test that anchor allocation scales with array size.
-
- Larger arrays should allocate proportionally more anchor slots
- to ensure adequate coverage across the data range.
- """
-
- @pytest.mark.parametrize(
- "size,max_items,expected_min_anchors",
- [
- (20, 10, 3), # Small array: at least 3 anchors (front + back)
- (100, 15, 4), # Medium array: at least 4 anchors
- (500, 20, 5), # Large array: at least 5 anchors
- (2000, 25, 6), # Very large: at least 6 anchors
- ],
- )
- def test_anchor_count_scales(self, tokenizer, size, max_items, expected_min_anchors):
- """Anchor count should increase with array size.
-
- Verifies that the number of items from boundary regions (front 10%, back 10%)
- increases as the array size grows.
- """
- items = generate_numbered_items(size)
-
- crushed = crush_items(items, tokenizer, max_items=max_items)
-
- # Count items from first 10% and last 10%
- front_boundary = int(size * 0.1)
- back_boundary = int(size * 0.9)
-
- anchor_count = sum(
- 1 for item in crushed if item["id"] < front_boundary or item["id"] >= back_boundary
- )
-
- assert anchor_count >= expected_min_anchors, (
- f"Array of size {size} should have at least {expected_min_anchors} "
- f"anchors from boundary regions, got {anchor_count}"
- )
-
- def test_small_array_high_preservation(self, tokenizer):
- """Small arrays (< max_items) should preserve most/all items.
-
- When the array is smaller than max_items, there's no need to drop items.
- """
- items = generate_numbered_items(8)
-
- # max_items=20 is larger than array size
- crushed = crush_items(items, tokenizer, max_items=20)
-
- # Should preserve all or nearly all items
- assert len(crushed) >= 7, f"Small array should preserve most items, got {len(crushed)}"
-
- def test_small_array_exact_size(self, tokenizer):
- """Array exactly at max_items should preserve all items."""
- items = generate_numbered_items(10)
-
- crushed = crush_items(items, tokenizer, max_items=10)
-
- # Should preserve all items
- assert len(crushed) == 10, "Array at max_items limit should keep all items"
-
- def test_large_array_efficient_sampling(self, tokenizer):
- """Large arrays should sample efficiently across all positions.
-
- A 500-item array crushed to 20 items should have representation
- from front, middle, and back regions.
- """
- items = generate_numbered_items(500)
-
- crushed = crush_items(items, tokenizer, max_items=20)
-
- positions = get_preserved_positions(crushed, 500)
-
- # Should have representation from all regions
- has_front = len(positions["front"]) >= 1
- has_back = len(positions["back"]) >= 1
- # Middle representation is optional but preferred for large arrays
- has_middle = len(positions["middle"]) >= 1
-
- assert has_front, "Large array should preserve items from front"
- assert has_back, "Large array should preserve items from back"
- # Middle coverage is important for large arrays
- assert has_middle, "Large array should have some middle representation"
-
- def test_very_large_array_coverage(self, tokenizer):
- """Very large array (1000+) should have good position distribution."""
- items = generate_numbered_items(1000)
-
- crushed = crush_items(items, tokenizer, max_items=25)
-
- positions = get_preserved_positions(crushed, 1000)
-
- # Calculate coverage spread
- all_positions = positions["front"] + positions["middle"] + positions["back"]
- if len(all_positions) >= 2:
- spread = max(all_positions) - min(all_positions)
- # Should span at least 80% of the array
- assert spread >= 800, f"Preserved items should span array, spread was {spread}"
-
-
-# =============================================================================
-# TestPatternAwareAnchoring
-# =============================================================================
-
-
-class TestPatternAwareAnchoring:
- """Test pattern-specific anchor strategies.
-
- Different data patterns (search results, logs, time series) should
- use different anchor weighting strategies.
- """
-
- def test_search_results_front_heavy(self, tokenizer):
- """Search results with scores should preserve more from front (high scores).
-
- Search results are sorted by relevance score, so top items are most important.
- """
- items = generate_search_results(100)
- for i, item in enumerate(items):
- item["idx"] = i # Track original position
-
- crushed = crush_items(items, tokenizer, max_items=12)
-
- # Count items by their original position
- front_count = sum(1 for item in crushed if item.get("idx", 100) < 30)
- back_count = sum(1 for item in crushed if item.get("idx", 0) >= 70)
-
- # For search results, front (high scores) should dominate
- assert front_count > back_count, (
- f"Search results should preserve more front items (high scores), "
- f"got front={front_count}, back={back_count}"
- )
-
- # Top results should definitely be present
- ids = [item["id"] for item in crushed]
- assert "doc_0" in ids, "Top search result should be preserved"
- assert "doc_1" in ids, "Second search result should be preserved"
-
- def test_logs_back_heavy(self, tokenizer):
- """Logs with timestamps should preserve more recent items (back of array).
-
- Logs are typically ordered chronologically, with recent items at the end.
- """
- items = generate_log_data(100)
- for i, item in enumerate(items):
- item["idx"] = i
-
- crushed = crush_items(items, tokenizer, max_items=12)
-
- # For logs, back (recent) should be emphasized
- # Extract indices - use 'idx' field we added
- indices = [item.get("idx", 0) for item in crushed]
-
- recent_count = sum(1 for idx in indices if idx >= 70)
- old_count = sum(1 for idx in indices if idx < 30)
-
- # Recent logs should be at least as represented as old logs
- assert recent_count >= old_count, (
- f"Logs should preserve recent items, got recent={recent_count}, old={old_count}"
- )
-
- def test_time_series_balanced(self, tokenizer):
- """Time series should have balanced front/back representation.
-
- For trend analysis, we need both the start and end of the time series.
- """
- items = generate_time_series_data(100)
- for i, item in enumerate(items):
- item["id"] = i
-
- crushed = crush_items(items, tokenizer, max_items=12)
-
- positions = get_preserved_positions(crushed, 100)
-
- front_count = len(positions["front"])
- back_count = len(positions["back"])
-
- # Should be relatively balanced for time series (within 2:1 ratio)
- if front_count > 0 and back_count > 0:
- ratio = max(front_count, back_count) / min(front_count, back_count)
- assert ratio <= 3, f"Time series should have balanced anchors, ratio was {ratio}"
-
- def test_generic_distributed(self, tokenizer):
- """Generic data should sample across all positions.
-
- When pattern is unknown, sampling should be distributed rather than
- heavily weighted to any particular region.
- """
- items = generate_numbered_items(100)
-
- crushed = crush_items(items, tokenizer, max_items=15)
-
- positions = get_preserved_positions(crushed, 100)
-
- # Should have items from multiple regions
- regions_with_items = sum(1 for region in ["front", "middle", "back"] if positions[region])
-
- assert regions_with_items >= 2, (
- f"Generic data should cover multiple regions, got {regions_with_items}"
- )
-
-
-# =============================================================================
-# TestQueryAwareAnchoring
-# =============================================================================
-
-
-class TestQueryAwareAnchoring:
- """Test query-based anchor adjustment.
-
- User queries containing temporal keywords should shift anchor weighting.
- """
-
- def test_latest_query_shifts_to_back(self, tokenizer):
- """'Latest' in query should preserve more recent items."""
- items = [{"id": i, "created": f"2024-01-{i:02d}"} for i in range(1, 31)]
-
- crushed = crush_items(
- items,
- tokenizer,
- max_items=8,
- query="Show me the latest entries",
- )
-
- ids = [item["id"] for item in crushed]
- recent_count = sum(1 for id in ids if id > 20)
-
- # Should have multiple recent items due to "latest" keyword
- assert recent_count >= 2, (
- f"Query with 'latest' should preserve recent items, got {recent_count}"
- )
-
- def test_recent_query_shifts_to_back(self, tokenizer):
- """'Recent' in query should preserve more recent items."""
- items = generate_log_data(50)
- for i, item in enumerate(items):
- item["idx"] = i
-
- crushed = crush_items(
- items,
- tokenizer,
- max_items=10,
- query="Show me recent log entries",
- )
-
- indices = [item.get("idx", 0) for item in crushed]
- recent_count = sum(1 for idx in indices if idx >= 35)
-
- assert recent_count >= 2, "Query with 'recent' should have multiple recent items"
-
- def test_first_query_shifts_to_front(self, tokenizer):
- """'First' in query should preserve earlier items."""
- items = [{"id": i, "created": f"2024-01-{i:02d}"} for i in range(1, 31)]
-
- crushed = crush_items(
- items,
- tokenizer,
- max_items=8,
- query="Show me the first entries",
- )
-
- ids = [item["id"] for item in crushed]
- early_count = sum(1 for id in ids if id < 10)
-
- # Should have multiple early items due to "first" keyword
- assert early_count >= 2, (
- f"Query with 'first' should preserve early items, got {early_count}"
- )
-
- def test_oldest_query_shifts_to_front(self, tokenizer):
- """'Oldest' in query should preserve earlier items."""
- items = generate_numbered_items(50)
-
- crushed = crush_items(
- items,
- tokenizer,
- max_items=10,
- query="Find the oldest records",
- )
-
- ids = [item["id"] for item in crushed]
- early_count = sum(1 for id in ids if id < 15)
-
- assert early_count >= 3, "Query with 'oldest' should have multiple early items"
-
- def test_specific_id_query_finds_item(self, tokenizer):
- """Query for specific ID should find it regardless of position."""
- items = [{"id": f"item_{i:04d}", "value": i} for i in range(200)]
-
- crushed = crush_items(
- items,
- tokenizer,
- max_items=10,
- query="Find item_0123",
- )
-
- # Item at position 123 should be found
- ids = [item["id"] for item in crushed]
- assert "item_0123" in ids, "Specific ID query should find the item"
-
- def test_no_query_uses_default_weights(self, tokenizer):
- """Without query, use pattern-based defaults."""
- items = generate_numbered_items(100)
-
- crushed = crush_items(items, tokenizer, max_items=15)
-
- # Without query, should use default anchoring (some front, some back)
- positions = get_preserved_positions(crushed, 100)
-
- assert len(positions["front"]) >= 1, "Default should include front items"
- assert len(positions["back"]) >= 1, "Default should include back items"
-
-
-# =============================================================================
-# TestInformationDensity
-# =============================================================================
-
-
-class TestInformationDensity:
- """Test information-density based selection.
-
- Items with unique or rare properties should be preferred over common ones.
- """
-
- def test_unique_items_preferred(self, tokenizer):
- """Items with rare field values should be preferred over common ones.
-
- When most items have status="ok" but a few have status="warning",
- the warning items should be preserved as they're more informative.
- """
- items = []
- for i in range(100):
- status = "warning" if i in [25, 50, 75] else "ok"
- items.append({"id": i, "status": status, "value": i})
-
- crushed = crush_items(items, tokenizer, max_items=15)
-
- # Warning items (rare status) should be preserved
- warning_count = sum(1 for item in crushed if item.get("status") == "warning")
- assert warning_count >= 2, f"Rare status items should be preferred, got {warning_count}"
-
- def test_dedup_identical_items(self, tokenizer):
- """Identical items should be deduplicated in anchor selection.
-
- If positions 0-5 all have identical content, we shouldn't keep all of them.
- """
- # First 6 items are identical
- items = [{"id": "dup", "value": 0, "constant": "same"} for _ in range(6)]
- # Rest are unique
- items.extend([{"id": f"uniq_{i}", "value": i * 10, "unique": True} for i in range(94)])
-
- crushed = crush_items(items, tokenizer, max_items=12)
-
- # Should not have many identical items
- dup_count = sum(1 for item in crushed if item.get("id") == "dup")
- unique_count = sum(1 for item in crushed if item.get("unique"))
-
- assert dup_count <= 2, f"Should deduplicate identical items, got {dup_count}"
- assert unique_count >= 6, f"Should prefer unique items, got {unique_count}"
-
- def test_structural_outliers_preferred(self, tokenizer):
- """Items with different structure should be preferred.
-
- An item with extra fields (like an error with stack trace) should be
- preferred over uniform items.
- """
- items = [{"id": i, "status": "ok"} for i in range(100)]
- # Add a structurally different item in the middle
- items[42] = {
- "id": 42,
- "status": "error",
- "error_message": "Something went wrong",
- "stack_trace": "at line 123...",
- "error_code": "ERR_001",
- }
-
- crushed = crush_items(items, tokenizer, max_items=15)
-
- # Structural outlier should be preserved
- outlier = [item for item in crushed if item.get("stack_trace")]
- assert len(outlier) >= 1, "Structurally different item should be preserved"
-
- def test_diverse_values_over_uniform(self, tokenizer):
- """When selecting from candidates, prefer diverse values."""
- items = []
- # Create items with varying diversity
- for i in range(100):
- items.append(
- {
- "id": i,
- "type": "common" if i < 90 else f"rare_type_{i}",
- "value": i,
- }
- )
-
- crushed = crush_items(items, tokenizer, max_items=15)
-
- # Should have some rare types
- rare_types = [item for item in crushed if "rare" in item.get("type", "")]
- assert len(rare_types) >= 1, "Rare type values should be preserved"
-
-
-# =============================================================================
-# TestCoverageMetrics
-# =============================================================================
-
-
-class TestCoverageMetrics:
- """Test that preserved items represent the full distribution.
-
- Compression should maintain coverage of value ranges, categories, and time.
- """
-
- def test_value_range_coverage(self, tokenizer):
- """Preserved items should cover the value range."""
- items = [{"id": i, "value": i} for i in range(100)]
-
- crushed = crush_items(items, tokenizer, max_items=12)
-
- values = [item["value"] for item in crushed]
-
- # Should cover most of the range [0, 100]
- assert min(values) < 10, "Should have low values"
- assert max(values) > 90, "Should have high values"
-
- # Should have some middle values too
- middle_count = sum(1 for v in values if 30 < v < 70)
- assert middle_count >= 1, "Should have some middle-range values"
-
- def test_category_coverage(self, tokenizer):
- """Preserved items should represent multiple categories."""
- items = generate_categorized_items({"A": 30, "B": 30, "C": 40})
-
- crushed = crush_items(items, tokenizer, max_items=12)
-
- categories = {item["category"] for item in crushed}
-
- # Should have at least 2 of 3 categories represented
- assert len(categories) >= 2, f"Should cover multiple categories, got {categories}"
-
- def test_category_proportional_representation(self, tokenizer):
- """Category distribution should roughly reflect original proportions."""
- items = generate_categorized_items({"major": 80, "minor": 20})
-
- crushed = crush_items(items, tokenizer, max_items=15)
-
- major_count = sum(1 for item in crushed if item["category"] == "major")
- minor_count = sum(1 for item in crushed if item["category"] == "minor")
-
- # Major category should have more items, but minor should be represented
- assert major_count > minor_count, "Major category should dominate"
- assert minor_count >= 1, "Minor category should still be represented"
-
- def test_temporal_coverage(self, tokenizer):
- """Preserved items should span the time range."""
- # Create items spanning 12 months
- items = [
- {"id": i, "timestamp": f"2024-{(i % 12) + 1:02d}-15", "event": f"event_{i}"}
- for i in range(60)
- ]
-
- crushed = crush_items(items, tokenizer, max_items=12)
-
- months = [int(item["timestamp"][5:7]) for item in crushed]
-
- # Should span a significant portion of the year
- month_range = max(months) - min(months)
- assert month_range >= 5, f"Should span multiple months, got range of {month_range}"
-
- def test_numeric_distribution_coverage(self, tokenizer):
- """Preserved numeric values should represent the distribution."""
- # Create items with bimodal distribution
- items = []
- for i in range(50):
- items.append({"id": i, "value": 10 + (i % 5)}) # Low cluster: 10-15
- for i in range(50):
- items.append({"id": 50 + i, "value": 90 + (i % 5)}) # High cluster: 90-95
-
- crushed = crush_items(items, tokenizer, max_items=12)
-
- values = [item["value"] for item in crushed]
-
- # Should have items from both clusters
- low_cluster = [v for v in values if v < 20]
- high_cluster = [v for v in values if v > 85]
-
- assert len(low_cluster) >= 1, "Should have items from low cluster"
- assert len(high_cluster) >= 1, "Should have items from high cluster"
-
-
-# =============================================================================
-# TestEdgeCases
-# =============================================================================
-
-
-class TestEdgeCases:
- """Edge cases and boundary conditions."""
-
- def test_empty_array(self, tokenizer):
- """Empty array should return empty result."""
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": "[]"},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- )
- crusher = SmartCrusher(config)
- result = crusher.apply(messages, tokenizer)
-
- tool_content = result.messages[-1]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- assert crushed == [], "Empty array should return empty result"
-
- def test_single_item(self, tokenizer):
- """Single item array should preserve the item."""
- items = [{"id": 0, "name": "Only item"}]
-
- crushed = crush_items(items, tokenizer, max_items=10, min_items=1)
-
- assert len(crushed) == 1, "Single item should be preserved"
- assert crushed[0]["name"] == "Only item"
-
- def test_two_items(self, tokenizer):
- """Two item array should preserve both items."""
- items = [{"id": 0, "name": "First"}, {"id": 1, "name": "Second"}]
-
- crushed = crush_items(items, tokenizer, max_items=10, min_items=1)
-
- assert len(crushed) == 2, "Both items should be preserved"
-
- def test_all_identical_items(self, tokenizer):
- """Array of all identical items should keep minimal anchors."""
- items = [{"id": "same", "value": 42, "status": "ok"} for _ in range(50)]
-
- crushed = crush_items(items, tokenizer, max_items=10)
-
- # Should not keep all 10 identical items - that would be wasteful
- # But some anchors are needed for structure
- assert len(crushed) <= 10, "Should respect max_items"
- # All items are identical, so any subset is equivalent
-
- def test_max_items_greater_than_array(self, tokenizer):
- """max_items > array size should return all items."""
- items = generate_numbered_items(8)
-
- crushed = crush_items(items, tokenizer, max_items=20)
-
- assert len(crushed) == 8, "Should return all items when max_items > array size"
-
- def test_max_items_equals_array(self, tokenizer):
- """max_items == array size should return all items."""
- items = generate_numbered_items(15)
-
- crushed = crush_items(items, tokenizer, max_items=15)
-
- assert len(crushed) == 15, "Should return all items when max_items == array size"
-
- def test_all_items_have_errors(self, tokenizer):
- """When all items have errors, quality guarantee preserves all.
-
- Current behavior: Error preservation takes precedence over max_items.
- All error items are preserved to avoid losing critical information.
- This is a deliberate design choice documented in SmartCrusher.
-
- NOTE: Future AnchorSelector may implement error deduplication or sampling
- for cases where all items are errors, but that requires careful design.
- """
- items = [{"id": i, "status": "error", "error_code": f"ERR_{i}"} for i in range(50)]
-
- crushed = crush_items(items, tokenizer, max_items=15)
-
- # Current behavior: all errors are preserved (quality guarantee)
- # This is intentional - we don't want to lose error information
- assert len(crushed) == 50, "All error items should be preserved (quality guarantee)"
-
- def test_none_values_handled(self, tokenizer):
- """Items with None values should be handled gracefully."""
- items = [
- {"id": i, "value": None if i % 3 == 0 else i * 10, "name": f"Item {i}"}
- for i in range(30)
- ]
-
- crushed = crush_items(items, tokenizer, max_items=10)
-
- # Should not crash, should return valid items
- assert len(crushed) > 0, "Should handle None values"
- assert all(isinstance(item, dict) for item in crushed)
-
- def test_mixed_types_in_array(self, tokenizer):
- """Array with mixed item structures should be handled."""
- items = [
- {"id": 0, "simple": True},
- {"id": 1, "nested": {"deep": {"value": 42}}},
- {"id": 2, "list_field": [1, 2, 3]},
- {"id": 3, "mixed": {"a": [1, 2], "b": "text"}},
- ]
- # Add more simple items to trigger crushing
- items.extend([{"id": i, "simple": True} for i in range(4, 20)])
-
- crushed = crush_items(items, tokenizer, max_items=8, min_items=3)
-
- # Should preserve structurally interesting items
- assert len(crushed) > 0, "Should handle mixed structures"
-
- def test_unicode_content_preserved(self, tokenizer):
- """Unicode content should be preserved correctly."""
- # Use the full Unicode codepoint for rocket emoji (U+1F680)
- rocket_emoji = "\U0001f680" # Full codepoint, not surrogate pair
- items = [
- {"id": i, "name": f"Item {i} - \u4e2d\u6587 \u65e5\u672c\u8a9e {rocket_emoji}"}
- for i in range(20)
- ]
-
- crushed = crush_items(items, tokenizer, max_items=10)
-
- # Unicode should be preserved
- for item in crushed:
- assert "\u4e2d\u6587" in item["name"], "Chinese characters should be preserved"
- assert rocket_emoji in item["name"], "Emoji should be preserved"
-
- def test_very_long_string_values(self, tokenizer):
- """Items with very long string values should be handled."""
- items = [{"id": i, "data": "x" * 10000 if i == 10 else "short"} for i in range(30)]
-
- crushed = crush_items(items, tokenizer, max_items=10)
-
- # Should not crash
- assert len(crushed) > 0, "Should handle long strings"
-
- def test_deeply_nested_items(self, tokenizer):
- """Deeply nested items should be handled without stack overflow."""
-
- def create_nested(depth: int) -> dict:
- if depth == 0:
- return {"value": "leaf"}
- return {"nested": create_nested(depth - 1)}
-
- items = [{"id": i, "deep": create_nested(10)} for i in range(20)]
-
- crushed = crush_items(items, tokenizer, max_items=10)
-
- assert len(crushed) > 0, "Should handle deeply nested items"
-
-
-# =============================================================================
-# TestAnchorConfigBehavior
-# =============================================================================
-
-
-class TestAnchorConfigBehavior:
- """Test configuration-driven anchor behavior."""
-
- def test_high_max_items_reduces_compression(self, tokenizer):
- """Higher max_items should preserve more items.
-
- NOTE: Data must have "importance signals" (errors, anomalies) to trigger
- compression. Generic unique items without signals are SKIPPED by the
- crushability analysis (conservative behavior to avoid losing entities).
-
- This test uses search results which always compress based on score.
- """
- # Use search results - they always trigger TOP_N compression
- items = generate_search_results(100)
-
- crushed_low = crush_items(items, tokenizer, max_items=8)
- crushed_high = crush_items(items, tokenizer, max_items=25)
-
- assert len(crushed_high) > len(crushed_low), (
- f"Higher max_items should preserve more items, "
- f"got low={len(crushed_low)}, high={len(crushed_high)}"
- )
-
- def test_min_items_to_analyze_threshold(self, tokenizer):
- """Arrays below min_items_to_analyze should not be crushed."""
- items = generate_numbered_items(5)
-
- # Set min_items_to_analyze higher than array size
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=10, # Higher than array size
- max_items_after_crush=3,
- )
- crusher = SmartCrusher(config)
- result = crusher.apply(messages, tokenizer)
-
- tool_content = result.messages[-1]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- # Should not crush because array is below threshold
- assert len(crushed) == 5, "Array below min_items_to_analyze should not be crushed"
-
-
-# =============================================================================
-# TestPreservationGuarantees
-# =============================================================================
-
-
-class TestPreservationGuarantees:
- """Test hard guarantees about what must be preserved."""
-
- def test_errors_always_preserved(self, tokenizer):
- """Error items must ALWAYS be preserved, regardless of position or count."""
- items = [{"id": i, "status": "ok"} for i in range(100)]
- # Scatter errors throughout
- error_positions = [10, 25, 45, 65, 85]
- for pos in error_positions:
- items[pos] = {"id": pos, "status": "error", "error": f"Error at {pos}"}
-
- crushed = crush_items(items, tokenizer, max_items=15)
-
- # All errors should be preserved
- error_count = sum(1 for item in crushed if item.get("status") == "error")
- assert error_count == len(error_positions), (
- f"All {len(error_positions)} errors should be preserved, got {error_count}"
- )
-
- def test_anomalies_always_preserved(self, tokenizer):
- """Numeric anomalies (statistical outliers) must be preserved."""
- items = [{"id": i, "value": 100 + (i % 5)} for i in range(100)]
- # Add anomalies
- items[33]["value"] = 9999
- items[66]["value"] = -9999
-
- crushed = crush_items(items, tokenizer, max_items=15)
-
- values = [item["value"] for item in crushed]
-
- # Anomalies should be present
- assert 9999 in values, "High anomaly should be preserved"
- assert -9999 in values, "Low anomaly should be preserved"
-
- def test_query_matches_always_preserved(self, tokenizer):
- """Items matching user query must be preserved."""
- items = [{"id": i, "name": f"generic_item_{i}"} for i in range(100)]
- items[55]["name"] = "special_target_item"
- items[55]["important"] = True
-
- crushed = crush_items(
- items,
- tokenizer,
- max_items=10,
- query="Find special_target_item",
- )
-
- # Target should be found
- names = [item["name"] for item in crushed]
- assert "special_target_item" in names, "Query-matched item must be preserved"
-
- def test_change_points_preserved(self, tokenizer):
- """Items around detected change points should be preserved."""
- # Create data with clear step change
- items = []
- for i in range(60):
- value = 100.0 if i < 30 else 300.0 # Step change at index 30
- items.append({"id": i, "value": value, "timestamp": f"2025-01-{(i % 28) + 1:02d}"})
-
- crushed = crush_items(items, tokenizer, max_items=12)
-
- # Should have items from around the change point
- preserved_ids = [item["id"] for item in crushed]
-
- # Check for items near the change point (indices 28-32)
- near_change = sum(1 for id in preserved_ids if 25 <= id <= 35)
- assert near_change >= 1, "Items near change point should be preserved"
diff --git a/tests/test_transforms/test_smart_crusher.py b/tests/test_transforms/test_smart_crusher.py
deleted file mode 100644
index 238690e7d..000000000
--- a/tests/test_transforms/test_smart_crusher.py
+++ /dev/null
@@ -1,1359 +0,0 @@
-"""Tests for SmartCrusher transform.
-
-Comprehensive tests covering:
-- SmartAnalyzer: Statistical analysis of arrays
-- SmartCrusher: Intelligent compression with Safe V1 Recipe
-- RelevanceScoring: Context extraction and item matching
-- Edge cases: Malformed JSON, nested arrays, different message formats
-"""
-
-import json
-
-import pytest
-
-from headroom import (
- OpenAIProvider,
- RelevanceScorerConfig,
- SmartCrusherConfig,
- Tokenizer,
-)
-from headroom.relevance import RelevanceScore, RelevanceScorer
-from headroom.transforms.smart_crusher import (
- CompressionStrategy,
- SmartAnalyzer,
- SmartCrusher,
-)
-
-# =============================================================================
-# Test Fixtures
-# =============================================================================
-
-# Create a shared provider for tests
-_provider = OpenAIProvider()
-
-
-def get_tokenizer(model: str = "gpt-4o") -> Tokenizer:
- """Get a tokenizer for tests using OpenAI provider."""
- token_counter = _provider.get_token_counter(model)
- return Tokenizer(token_counter, model)
-
-
-def tokenizer():
- """Provide a tokenizer for tests."""
- return get_tokenizer()
-
-
-def default_config():
- """Default SmartCrusherConfig for testing."""
- return SmartCrusherConfig(
- enabled=True,
- min_items_to_analyze=3,
- min_tokens_to_crush=0, # Always crush for tests
- max_items_after_crush=10,
- variance_threshold=2.0,
- )
-
-
-def analyzer(default_config):
- """SmartAnalyzer instance for testing."""
- return SmartAnalyzer(default_config)
-
-
-def crusher(default_config):
- """SmartCrusher instance for testing."""
- return SmartCrusher(default_config)
-
-
-# =============================================================================
-# Test Data Generators
-# =============================================================================
-
-
-def generate_time_series_data(n: int = 20, with_spike: bool = False) -> list[dict]:
- """Generate time series data with optional anomaly."""
- data = []
- for i in range(n):
- value = 100.0 + (i * 0.5) # Slight upward trend
- if with_spike and i == n // 2:
- value = 500.0 # Spike in the middle
- data.append(
- {
- "timestamp": f"2025-01-{(i % 28) + 1:02d}T12:00:00Z",
- "value": value,
- "metric": "cpu_usage",
- }
- )
- return data
-
-
-def generate_log_data(n: int = 20, with_errors: bool = False) -> list[dict]:
- """Generate log-style data with optional errors."""
- data = []
- levels = ["INFO", "DEBUG", "WARN"]
- for i in range(n):
- level = levels[i % len(levels)]
- if with_errors and i in [5, 15]:
- level = "ERROR"
- message = f"Connection failed: timeout after 30s (attempt {i})"
- else:
- message = f"Processing request {i} successfully"
- data.append(
- {
- "level": level,
- "message": message,
- "timestamp": f"2025-01-06T{12 + (i // 60):02d}:{i % 60:02d}:00Z",
- }
- )
- return data
-
-
-def generate_search_results(n: int = 20) -> list[dict]:
- """Generate search results with scores."""
- return [
- {
- "id": f"doc_{i}",
- "title": f"Document {i}",
- "score": 1.0 - (i * 0.05),
- "snippet": f"This is a snippet from document {i}...",
- }
- for i in range(n)
- ]
-
-
-def generate_generic_data(
- n: int = 20,
- constant_field: bool = False,
- with_signals: bool = False,
-) -> list[dict]:
- """Generate generic array data.
-
- Args:
- n: Number of items to generate
- constant_field: If True, type field is constant "product"
- with_signals: If True, adds importance signals (errors, anomalies)
- to enable crushing with new statistical detection
- """
- items = []
- for i in range(n):
- item = {
- "id": i,
- "name": f"Item {i}",
- "type": "product" if constant_field else f"type_{i % 3}",
- "active": True if constant_field else (i % 2 == 0),
- }
- if with_signals:
- item["value"] = 100.0
- # Add some errors
- if i == n // 4:
- item["error"] = f"Error at {i}"
- # Add some anomalies
- if i == n // 2:
- item["value"] = 99999.0
- items.append(item)
- return items
-
-
-# =============================================================================
-# TestSmartAnalyzer
-# =============================================================================
-
-
-class TestSmartAnalyzer:
- """Tests for SmartAnalyzer class."""
-
- def test_analyze_empty_array(self, analyzer):
- """Empty array should return analysis with no field stats."""
- result = analyzer.analyze_array([])
-
- assert result.item_count == 0
- assert result.field_stats == {}
- assert result.detected_pattern == "generic"
- assert result.recommended_strategy == CompressionStrategy.NONE
- assert result.constant_fields == {}
-
- def test_analyze_single_item(self, analyzer):
- """Single item array should return analysis but no compression."""
- items = [{"id": 1, "name": "Test"}]
- result = analyzer.analyze_array(items)
-
- assert result.item_count == 1
- assert "id" in result.field_stats
- assert "name" in result.field_stats
- # Single item means constant fields
- assert result.field_stats["id"].is_constant
- assert result.field_stats["name"].is_constant
-
- def test_analyze_numeric_field_stats(self, analyzer):
- """Numeric fields should have correct statistics computed."""
- items = [
- {"value": 10.0},
- {"value": 20.0},
- {"value": 30.0},
- {"value": 40.0},
- {"value": 50.0},
- ]
- result = analyzer.analyze_array(items)
-
- stats = result.field_stats["value"]
- assert stats.field_type == "numeric"
- assert stats.min_val == 10.0
- assert stats.max_val == 50.0
- assert stats.mean_val == 30.0
- assert stats.variance is not None
- assert stats.variance > 0
-
- def test_analyze_string_field_stats(self, analyzer):
- """String fields should have correct statistics computed."""
- items = [
- {"name": "Alice"},
- {"name": "Bob"},
- {"name": "Alice"}, # Duplicate
- {"name": "Charlie"},
- {"name": "Alice"}, # Another duplicate
- ]
- result = analyzer.analyze_array(items)
-
- stats = result.field_stats["name"]
- assert stats.field_type == "string"
- assert stats.avg_length is not None
- assert stats.top_values is not None
- # Alice appears 3 times, should be top
- assert stats.top_values[0][0] == "Alice"
- assert stats.top_values[0][1] == 3
-
- def test_detect_time_series_pattern(self, analyzer):
- """Time series data should be detected correctly."""
- # Create data with timestamp and numeric variance
- # Include anomaly to provide an importance signal for crushing
- items = []
- for i in range(40):
- # Create variance-inducing data
- value = 100.0 + (i * 2.0) # Steady increase with variance
- if i == 20:
- value = 999.0 # Anomaly provides importance signal
- items.append(
- {
- "timestamp": f"2025-01-{(i % 28) + 1:02d}T12:00:00Z",
- "value": value,
- "metric": "cpu_usage",
- }
- )
-
- result = analyzer.analyze_array(items)
-
- # Pattern should be detected as time_series (timestamp + numeric variance)
- assert result.detected_pattern == "time_series"
- # With anomaly signal, strategy should allow crushing
- assert result.recommended_strategy in [
- CompressionStrategy.TIME_SERIES,
- CompressionStrategy.SMART_SAMPLE,
- ]
-
- def test_detect_time_series_pattern_with_change_points(self):
- """Time series with clear change points should use TIME_SERIES strategy."""
- # The change point detection threshold is variance_threshold * std
- # To detect a change point, the before/after mean difference must exceed this
- # With bimodal data, std is very high. We need a lower variance_threshold
- # to reliably detect change points, OR the test should use a config
- # with lower variance threshold.
-
- config = SmartCrusherConfig(
- min_items_to_analyze=3,
- variance_threshold=1.0, # Lower threshold to detect changes
- )
- analyzer = SmartAnalyzer(config)
-
- # Create data with clear step change
- items = []
- for i in range(40):
- if i < 20:
- value = 100.0 + (i * 0.5) # Values around 100-110
- else:
- value = 300.0 + ((i - 20) * 0.5) # Values around 300-310 (jump)
- items.append(
- {
- "timestamp": f"2025-01-{(i % 28) + 1:02d}T12:00:00Z",
- "value": value,
- "metric": "cpu_usage",
- }
- )
-
- result = analyzer.analyze_array(items)
-
- assert result.detected_pattern == "time_series"
- # With lower variance_threshold, change points should be detected
- value_stats = result.field_stats.get("value")
- assert value_stats is not None
- # Even with low threshold, bimodal data has high std
- # The test verifies the strategy selection logic
- if len(value_stats.change_points) > 0:
- assert result.recommended_strategy == CompressionStrategy.TIME_SERIES
- else:
- # If change points still not detected, strategy falls back
- assert result.recommended_strategy in [
- CompressionStrategy.TIME_SERIES,
- CompressionStrategy.SMART_SAMPLE,
- ]
-
- def test_detect_logs_pattern(self, analyzer):
- """Log data should be detected correctly."""
- # Use logs WITH errors to provide importance signal
- items = generate_log_data(20, with_errors=True)
- result = analyzer.analyze_array(items)
-
- # With structural detection, logs are detected as logs pattern
- # but strategy depends on crushability analysis
- assert result.detected_pattern in ["logs", "generic"]
- # With error items providing signal, crushing can proceed
- assert result.recommended_strategy in [
- CompressionStrategy.CLUSTER_SAMPLE,
- CompressionStrategy.SMART_SAMPLE,
- CompressionStrategy.SKIP, # May still skip if other conditions met
- ]
-
- def test_detect_search_results_pattern(self, analyzer):
- """Search results with scores should be detected correctly."""
- items = generate_search_results(20)
- result = analyzer.analyze_array(items)
-
- assert result.detected_pattern == "search_results"
- assert result.recommended_strategy == CompressionStrategy.TOP_N
-
- def test_detect_generic_pattern(self, analyzer):
- """Generic data without special patterns should be detected."""
- items = generate_generic_data(20)
- result = analyzer.analyze_array(items)
-
- assert result.detected_pattern == "generic"
- # With new crushability analysis: unique IDs + no importance signal = SKIP
- # This is the safe behavior to avoid dropping important unique entities
- assert result.recommended_strategy in [
- CompressionStrategy.SMART_SAMPLE,
- CompressionStrategy.SKIP, # More conservative when no signal present
- ]
-
- def test_detect_change_points(self, analyzer):
- """Change points should be detected in numeric data with variance."""
- # Create data with clear change point
- items = []
- for i in range(30):
- if i < 15:
- value = 100.0 + (i * 0.1) # Low values
- else:
- value = 200.0 + ((i - 15) * 0.1) # High values after change
- items.append({"timestamp": f"2025-01-{(i % 28) + 1:02d}", "metric": value})
-
- result = analyzer.analyze_array(items)
-
- # Should detect change point around index 15
- metric_stats = result.field_stats.get("metric")
- assert metric_stats is not None
- assert metric_stats.change_points is not None
- # Change points should be near the transition
- if metric_stats.change_points:
- assert any(10 <= cp <= 20 for cp in metric_stats.change_points)
-
- def test_constant_field_detection(self, analyzer):
- """Constant fields should be identified."""
- items = generate_generic_data(20, constant_field=True)
- result = analyzer.analyze_array(items)
-
- # type field should be constant ("product")
- type_stats = result.field_stats.get("type")
- assert type_stats is not None
- assert type_stats.is_constant
- assert type_stats.constant_value == "product"
-
- # Constant fields should be in constant_fields dict
- assert "type" in result.constant_fields
- assert result.constant_fields["type"] == "product"
-
-
-# =============================================================================
-# TestSmartCrusher
-# =============================================================================
-
-
-class TestSmartCrusher:
- """Tests for SmartCrusher transform."""
-
- def test_should_apply_below_threshold(self, tokenizer):
- """Should not apply when tokens below min_tokens_to_crush."""
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": '{"status": "ok"}'},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=1000, # High threshold
- )
- crusher = SmartCrusher(config)
-
- assert not crusher.should_apply(messages, tokenizer)
-
- def test_should_apply_no_arrays(self, tokenizer):
- """Should not apply when no crushable arrays present."""
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": '{"status": "ok", "value": 123}'},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- )
- crusher = SmartCrusher(config)
-
- assert not crusher.should_apply(messages, tokenizer)
-
- def test_should_apply_small_array(self, tokenizer):
- """Should not apply when array is below min_items_to_analyze."""
- small_array = [{"id": i} for i in range(3)]
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(small_array)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=10, # Array too small
- )
- crusher = SmartCrusher(config)
-
- assert not crusher.should_apply(messages, tokenizer)
-
- def test_crush_time_series_keeps_change_points(self, tokenizer, default_config):
- """Time series crushing should preserve items around change points."""
- # Create data with clear change point AND an anomaly signal
- items = []
- for i in range(30):
- if i < 15:
- value = 100.0
- else:
- value = 200.0 # Jump at index 15
- # Add anomaly to provide importance signal for crushing
- if i == 25:
- value = 999.0 # Extreme anomaly
- items.append(
- {
- "timestamp": f"2025-01-{(i % 28) + 1:02d}T12:00:00Z",
- "value": value,
- }
- )
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=15,
- preserve_change_points=True,
- )
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Parse result
- tool_content = result.messages[1]["content"]
- # Remove digest marker
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- # Should have fewer items than original
- assert len(crushed) < len(items)
-
- # Should have items around change point (index ~15)
- values = [item["value"] for item in crushed]
- # Should have both low and high values (around change point)
- assert 100.0 in values
- assert 200.0 in values
-
- def test_crush_keeps_first_k_items(self, tokenizer):
- """Crushing should always keep first K items (Safe V1 Recipe)."""
- items = generate_generic_data(30)
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- )
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Parse result
- tool_content = result.messages[1]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- # First 3 items should be preserved
- crushed_ids = [item["id"] for item in crushed]
- assert 0 in crushed_ids # First item
- assert 1 in crushed_ids # Second item
- assert 2 in crushed_ids # Third item
-
- def test_crush_keeps_last_k_items(self, tokenizer):
- """Crushing should always keep last K items (Safe V1 Recipe)."""
- items = generate_generic_data(30)
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- )
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Parse result
- tool_content = result.messages[1]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- # Last 2 items should be preserved
- crushed_ids = [item["id"] for item in crushed]
- assert 28 in crushed_ids # Second to last
- assert 29 in crushed_ids # Last item
-
- def test_crush_keeps_error_items(self, tokenizer):
- """Crushing should always preserve error items (Safe V1 Recipe)."""
- items = generate_generic_data(30)
- # Add error items in the middle
- items[10] = {"id": 10, "status": "error", "message": "Connection failed"}
- items[20] = {"id": 20, "status": "failed", "exception": "TimeoutError"}
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- )
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Parse result
- tool_content = result.messages[1]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- # Error items should be preserved
- crushed_ids = [item["id"] for item in crushed]
- assert 10 in crushed_ids # Error item
- assert 20 in crushed_ids # Failed item
-
- def test_crush_keeps_anomalies(self, tokenizer):
- """Crushing should preserve anomalous numeric items (> 2 std from mean)."""
- items = []
- for i in range(30):
- value = 100.0 + (i * 0.1) # Normal range ~100-103
- items.append({"id": i, "metric": value})
-
- # Add anomaly in the middle
- items[15]["metric"] = 500.0 # Way above mean
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- variance_threshold=2.0,
- )
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Parse result
- tool_content = result.messages[1]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- # Anomaly should be preserved
- crushed_ids = [item["id"] for item in crushed]
- assert 15 in crushed_ids # Anomaly
-
- def test_crush_top_n_by_score(self, tokenizer):
- """Search results should be crushed keeping top N by score."""
- items = generate_search_results(30)
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- )
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Parse result
- tool_content = result.messages[1]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- # Should have reduced items
- assert len(crushed) < len(items)
-
- # Top scored items should be present
- crushed_ids = [item["id"] for item in crushed]
- assert "doc_0" in crushed_ids # Highest score
- assert "doc_1" in crushed_ids # Second highest
-
- def test_schema_preserved(self, tokenizer):
- """Output items should have same schema as input items."""
- items = [
- {
- "id": i,
- "name": f"Item {i}",
- "nested": {"key": f"value_{i}"},
- "tags": ["a", "b"],
- }
- for i in range(20)
- ]
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- )
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Parse result
- tool_content = result.messages[1]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- # Each item should have original schema
- original_keys = set(items[0].keys())
- for item in crushed:
- assert set(item.keys()) == original_keys
- assert "nested" in item
- assert "key" in item["nested"]
- assert "tags" in item
- assert isinstance(item["tags"], list)
-
- def test_respects_max_items_after_crush(self, tokenizer):
- """Output should respect max_items_after_crush limit."""
- # Create data with importance signals (errors, anomalies) so crushing happens
- items = []
- for i in range(100):
- item = {
- "id": i,
- "name": f"Item {i}",
- "type": f"type_{i % 3}",
- "value": 100.0,
- }
- # Add some errors to provide importance signal
- if i in [10, 30, 50, 70, 90]:
- item["error"] = f"Error at {i}"
- # Add some anomalies
- if i in [15, 45, 75]:
- item["value"] = 99999.0
- items.append(item)
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=15,
- )
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Parse result
- tool_content = result.messages[1]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- # With errors and anomalies, crushing should happen
- # But critical items override max, so we may have more than 15
- # The test verifies that crushing happened (fewer than original)
- assert len(crushed) < 100, "Should compress the data"
- # Errors must be preserved
- error_count = sum(1 for x in crushed if x.get("error"))
- assert error_count == 5, "All errors must be preserved"
-
-
-# =============================================================================
-# TestRelevanceScoring
-# =============================================================================
-
-
-class TestRelevanceScoring:
- """Tests for relevance scoring in SmartCrusher."""
-
- def test_context_extraction_from_user_messages(self, tokenizer):
- """Context should be extracted from user messages."""
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "user", "content": "Find user Alice in the database"},
- {
- "role": "assistant",
- "content": None,
- "tool_calls": [
- {
- "id": "call_1",
- "type": "function",
- "function": {"name": "search_users", "arguments": "{}"},
- }
- ],
- },
- {"role": "tool", "tool_call_id": "call_1", "content": "[]"},
- ]
-
- config = SmartCrusherConfig(enabled=True, min_tokens_to_crush=0)
- crusher = SmartCrusher(config)
-
- context = crusher._extract_context_from_messages(messages)
-
- assert "Alice" in context
- assert "database" in context
-
- def test_context_extraction_from_tool_calls(self, tokenizer):
- """Context should be extracted from tool call arguments."""
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "user", "content": "Search for it"},
- {
- "role": "assistant",
- "content": None,
- "tool_calls": [
- {
- "id": "call_1",
- "type": "function",
- "function": {
- "name": "search_users",
- "arguments": '{"query": "user_id=12345"}',
- },
- }
- ],
- },
- {"role": "tool", "tool_call_id": "call_1", "content": "[]"},
- ]
-
- config = SmartCrusherConfig(enabled=True, min_tokens_to_crush=0)
- crusher = SmartCrusher(config)
-
- context = crusher._extract_context_from_messages(messages)
-
- assert "12345" in context
-
- def test_relevance_keeps_matching_items(self, tokenizer):
- """Items matching user query should be preserved."""
- # Create items where one matches the user query
- items = [{"id": i, "name": f"User {i}", "email": f"user{i}@example.com"} for i in range(30)]
- # Add special user Alice
- items[15] = {"id": 15, "name": "Alice", "email": "[email protected]"}
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "user", "content": "Find user Alice"},
- {
- "role": "assistant",
- "content": None,
- "tool_calls": [
- {
- "id": "call_1",
- "type": "function",
- "function": {"name": "list_users", "arguments": "{}"},
- }
- ],
- },
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- # Use BM25 scorer for deterministic testing
- relevance_config = RelevanceScorerConfig(
- tier="bm25",
- relevance_threshold=0.1, # Lower threshold to catch matches
- )
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- )
- crusher = SmartCrusher(config, relevance_config=relevance_config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Parse result
- tool_content = result.messages[-1]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- # Alice should be preserved due to relevance matching
- crushed_names = [item["name"] for item in crushed]
- assert "Alice" in crushed_names
-
- def test_custom_scorer_injection(self, tokenizer):
- """Custom scorer should be used when provided."""
-
- class MockScorer(RelevanceScorer):
- """Mock scorer that always returns high score for items with id=5."""
-
- def score(self, item: str, context: str) -> RelevanceScore:
- if '"id": 5' in item or '"id":5' in item:
- return RelevanceScore(score=0.9, reason="mock high score")
- return RelevanceScore(score=0.0, reason="mock low score")
-
- def score_batch(self, items: list[str], context: str) -> list[RelevanceScore]:
- return [self.score(item, context) for item in items]
-
- items = generate_generic_data(30)
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "user", "content": "Find the special item"},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- )
- crusher = SmartCrusher(config, scorer=MockScorer())
-
- result = crusher.apply(messages, tokenizer)
-
- # Parse result
- tool_content = result.messages[-1]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- # Item with id=5 should be preserved due to mock scorer
- crushed_ids = [item["id"] for item in crushed]
- assert 5 in crushed_ids
-
-
-# =============================================================================
-# TestEdgeCases
-# =============================================================================
-
-
-class TestEdgeCases:
- """Edge case tests for SmartCrusher."""
-
- def test_malformed_json_passthrough(self, tokenizer):
- """Malformed JSON should pass through unchanged."""
- malformed = "{ this is not valid JSON ["
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": malformed},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- )
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Content should be unchanged
- assert result.messages[1]["content"] == malformed
-
- def test_nested_arrays(self, tokenizer):
- """Nested arrays should be handled correctly."""
- # Use with_signals=True to enable crushing with new statistical detection
- nested_data = {
- "results": generate_generic_data(20, with_signals=True),
- "metadata": {
- "inner_array": [{"x": i, "value": 100.0 if i != 7 else 99999.0} for i in range(15)],
- },
- }
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(nested_data)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- )
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Should be modified
- tool_content = result.messages[1]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- # Results array should be crushed (with signals, crushing can happen)
- assert len(crushed["results"]) < 20, "Results should be crushed"
-
- # Nested array should also be crushed if large enough
- assert "metadata" in crushed
- assert "inner_array" in crushed["metadata"]
-
- def test_anthropic_style_tool_results(self, tokenizer):
- """Anthropic-style tool_result blocks should be handled."""
- # Use with_signals=True to enable crushing with new statistical detection
- items = generate_generic_data(20, with_signals=True)
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {
- "role": "user",
- "content": [
- {"type": "text", "text": "What are the results?"},
- {
- "type": "tool_result",
- "tool_use_id": "toolu_123",
- "content": json.dumps(items),
- },
- ],
- },
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- )
- crusher = SmartCrusher(config)
-
- # Check should_apply works
- assert crusher.should_apply(messages, tokenizer)
-
- result = crusher.apply(messages, tokenizer)
-
- # Tool result content should be crushed (with signals present)
- tool_result_block = result.messages[1]["content"][1]
- content = tool_result_block["content"]
- json_part = content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- assert len(crushed) < len(items), "With signals, crushing should happen"
-
- def test_openai_style_tool_results(self, tokenizer):
- """OpenAI-style tool messages should be handled."""
- # Use with_signals=True to enable crushing with new statistical detection
- items = generate_generic_data(20, with_signals=True)
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "user", "content": "Search for items"},
- {
- "role": "assistant",
- "content": None,
- "tool_calls": [
- {
- "id": "call_abc123",
- "type": "function",
- "function": {"name": "search_items", "arguments": "{}"},
- }
- ],
- },
- {
- "role": "tool",
- "tool_call_id": "call_abc123",
- "content": json.dumps(items),
- },
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- )
- crusher = SmartCrusher(config)
-
- # Check should_apply works
- assert crusher.should_apply(messages, tokenizer)
-
- result = crusher.apply(messages, tokenizer)
-
- # Tool content should be crushed
- tool_content = result.messages[3]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- assert len(crushed) < len(items)
- assert len(crushed) <= 10
-
- def test_empty_tool_content(self, tokenizer):
- """Empty tool content should be handled gracefully."""
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": ""},
- ]
-
- config = SmartCrusherConfig(enabled=True, min_tokens_to_crush=0)
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Should not crash, content unchanged
- assert result.messages[1]["content"] == ""
-
- def test_non_dict_array_items(self, tokenizer):
- """Arrays of non-dict items should be handled gracefully."""
- items = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- )
- crusher = SmartCrusher(config)
-
- # Should not crash, but won't crush non-dict arrays
- result = crusher.apply(messages, tokenizer)
-
- # Primitive arrays are not crushed by SmartCrusher (it requires dict items)
- # Content should pass through
- assert result.messages is not None
-
- def test_mixed_null_values(self, tokenizer):
- """Items with null values should be handled without crashing.
-
- With statistical detection, this data has unique IDs and no importance
- signals, so it will be SKIPPED (not crushed) - which is correct behavior.
- The test verifies that null values don't crash the analyzer.
- """
- items = [{"id": i, "value": None if i % 2 == 0 else i * 10} for i in range(20)]
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- )
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Should not crash - parsing should succeed
- tool_content = result.messages[1]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- # With statistical detection, unique entities with no signals are SKIPPED
- # This is correct conservative behavior - all 20 items preserved
- assert len(crushed) == 20
-
- def test_unicode_content(self, tokenizer):
- """Unicode content should be preserved."""
- items = [
- {"id": i, "name": f"Item {i} - \u4e2d\u6587 \u65e5\u672c\u8a9e \ud83d\ude80"}
- for i in range(20)
- ]
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {
- "role": "tool",
- "tool_call_id": "call_1",
- "content": json.dumps(items, ensure_ascii=False),
- },
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- )
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- tool_content = result.messages[1]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- # Unicode should be preserved
- for item in crushed:
- assert "\u4e2d\u6587" in item["name"]
- assert "\u65e5\u672c\u8a9e" in item["name"]
-
- def test_digest_marker_added(self, tokenizer):
- """Digest marker should be added to crushed content."""
- items = generate_generic_data(30)
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- )
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- tool_content = result.messages[1]["content"]
-
- # Should have digest marker
- assert "<headroom:tool_digest" in tool_content
- assert "sha256=" in tool_content
- assert len(result.markers_inserted) > 0
-
- def test_transforms_applied_tracking(self, tokenizer):
- """Transforms applied should be tracked."""
- items = generate_generic_data(30)
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- )
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Should track transforms
- assert len(result.transforms_applied) > 0
- assert any("smart" in t.lower() for t in result.transforms_applied)
-
- def test_token_reduction(self, tokenizer):
- """Token count should be reduced after crushing."""
- items = generate_generic_data(100)
-
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- )
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Tokens should be reduced
- assert result.tokens_after < result.tokens_before
-
-
-# =============================================================================
-# Integration Tests
-# =============================================================================
-
-
-class TestSmartCrusherIntegration:
- """Integration tests for SmartCrusher with realistic scenarios."""
-
- def test_database_query_results(self, tokenizer):
- """Simulate crushing database query results."""
- # Simulate a database query returning many rows
- items = [
- {
- "user_id": f"usr_{i:05d}",
- "email": f"user{i}@example.com",
- "created_at": f"2025-01-{(i % 28) + 1:02d}T00:00:00Z",
- "status": "active" if i % 10 != 0 else "inactive",
- "login_count": i * 5,
- }
- for i in range(100)
- ]
-
- messages = [
- {"role": "system", "content": "You are a database assistant."},
- {"role": "user", "content": "Show me users with email containing 'user50'"},
- {
- "role": "assistant",
- "tool_calls": [
- {
- "id": "call_1",
- "type": "function",
- "function": {"name": "query_users", "arguments": "{}"},
- }
- ],
- },
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- relevance_config = RelevanceScorerConfig(tier="bm25", relevance_threshold=0.1)
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=15,
- )
- crusher = SmartCrusher(config, relevance_config=relevance_config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Parse result
- tool_content = result.messages[3]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- # Should be significantly reduced
- assert len(crushed) <= 15
- assert len(crushed) < len(items)
-
- # User 50 should be preserved due to relevance
- user_ids = [item["user_id"] for item in crushed]
- assert "usr_00050" in user_ids
-
- def test_api_search_results(self, tokenizer):
- """Simulate crushing API search results."""
- items = [
- {
- "id": f"result_{i}",
- "title": f"Result {i}: {'Important Finding' if i < 5 else 'Regular Result'}",
- "relevance_score": 0.95 - (i * 0.02),
- "snippet": f"This is the snippet for result {i}...",
- "url": f"https://example.com/doc/{i}",
- }
- for i in range(50)
- ]
-
- messages = [
- {"role": "system", "content": "You are a search assistant."},
- {"role": "user", "content": "Search for important findings"},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=10,
- )
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Parse result
- tool_content = result.messages[2]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- # Should use TOP_N strategy
- assert len(crushed) <= 12 # max + some buffer for first/last
-
- # Top results should be preserved
- ids = [item["id"] for item in crushed]
- assert "result_0" in ids
- assert "result_1" in ids
-
- def test_monitoring_metrics(self, tokenizer):
- """Simulate crushing monitoring/metrics data."""
- items = []
- for i in range(60):
- # Normal CPU usage around 50%
- cpu = 50.0 + (i * 0.1)
- # Spike at index 30
- if i == 30:
- cpu = 95.0
- items.append(
- {
- "timestamp": f"2025-01-06T{10 + (i // 60):02d}:{i % 60:02d}:00Z",
- "cpu_percent": cpu,
- "memory_percent": 60.0,
- "host": "server-01",
- }
- )
-
- messages = [
- {"role": "system", "content": "You are a monitoring assistant."},
- {"role": "user", "content": "Show me CPU metrics"},
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(items)},
- ]
-
- config = SmartCrusherConfig(
- enabled=True,
- min_tokens_to_crush=0,
- min_items_to_analyze=3,
- max_items_after_crush=15,
- preserve_change_points=True,
- )
- crusher = SmartCrusher(config)
-
- result = crusher.apply(messages, tokenizer)
-
- # Parse result
- tool_content = result.messages[2]["content"]
- json_part = tool_content.split("\n<headroom:")[0]
- crushed = json.loads(json_part)
-
- # Should preserve the spike
- cpu_values = [item["cpu_percent"] for item in crushed]
- assert 95.0 in cpu_values # Spike should be preserved
diff --git a/tests/test_transforms/test_smart_crusher_bugs.py b/tests/test_transforms/test_smart_crusher_bugs.py
index e09f2deee..3e7aa22e5 100644
--- a/tests/test_transforms/test_smart_crusher_bugs.py
+++ b/tests/test_transforms/test_smart_crusher_bugs.py
@@ -20,6 +20,7 @@ from headroom.transforms.smart_crusher import SmartCrusher
def _make_crusher(max_items: int = 10, min_items: int = 3) -> SmartCrusher:
+ """Build a SmartCrusher with deterministic small-K config for tests."""
config = SmartCrusherConfig(
enabled=True,
min_items_to_analyze=min_items,
@@ -30,102 +31,13 @@ def _make_crusher(max_items: int = 10, min_items: int = 3) -> SmartCrusher:
return SmartCrusher(config=config)
-# ---------------------------------------------------------------------------
-# Bug 1: Number array type mixing
-# ---------------------------------------------------------------------------
-
-
-class TestNumberArraySchemaPreservation:
- """_crush_number_array must return only original numeric values.
-
- Previously it prepended a stats summary string, producing
- [string, int, int, ...] which violates the schema-preserving
- guarantee and breaks type-aware JSON consumers.
- """
-
- def test_crushed_number_array_contains_only_numbers(self) -> None:
- """Every element of the crushed array must be int or float."""
- crusher = _make_crusher(max_items=10)
- numbers = list(range(50)) # 0..49, well above the n<=8 passthrough
- crushed, strategy = crusher._crush_number_array(numbers)
-
- for i, item in enumerate(crushed):
- assert isinstance(item, int | float), (
- f"Item {i} is {type(item).__name__} = {item!r}, expected int/float. "
- f"Schema-preserving guarantee violated."
- )
-
- def test_crushed_number_array_subset_of_original(self) -> None:
- """Every value in the crushed array must exist in the original."""
- crusher = _make_crusher(max_items=10)
- numbers = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]
- crushed, _ = crusher._crush_number_array(numbers)
-
- original_set = set(numbers)
- for item in crushed:
- assert item in original_set, (
- f"Value {item!r} not in original array — generated content detected"
- )
-
- def test_stats_summary_in_strategy_not_in_array(self) -> None:
- """Statistics should be communicated via strategy string, not array content."""
- crusher = _make_crusher(max_items=5)
- numbers = list(range(100))
- crushed, strategy = crusher._crush_number_array(numbers)
-
- # Strategy should contain stats info
- assert "number:" in strategy
-
- # Array should not contain any strings
- strings_in_result = [x for x in crushed if isinstance(x, str)]
- assert strings_in_result == [], f"Found string(s) in numeric array: {strings_in_result}"
-
- def test_number_array_passthrough_for_small(self) -> None:
- """Arrays with n <= 8 should pass through unchanged."""
- crusher = _make_crusher()
- small = [1, 2, 3, 4, 5]
- crushed, strategy = crusher._crush_number_array(small)
- assert crushed == small
- assert strategy == "number:passthrough"
-
- def test_number_array_preserves_outliers(self) -> None:
- """Outlier values should be preserved in the crushed output."""
- crusher = _make_crusher(max_items=10)
- # Normal range + extreme outlier
- numbers = [10] * 20 + [10000]
- crushed, strategy = crusher._crush_number_array(numbers)
- assert 10000 in crushed, "Outlier value 10000 was dropped"
-
- def test_number_array_preserves_boundaries(self) -> None:
- """First and last values should always be kept."""
- crusher = _make_crusher(max_items=5)
- numbers = list(range(100))
- crushed, strategy = crusher._crush_number_array(numbers)
- assert crushed[0] == 0, "First value not preserved"
- assert numbers[-1] in crushed, "Last value not preserved"
-
- def test_non_finite_passthrough(self) -> None:
- """All-NaN/Inf arrays should return unchanged."""
- crusher = _make_crusher()
- nans = [float("nan")] * 10
- crushed, strategy = crusher._crush_number_array(nans)
- assert strategy == "number:no_finite"
- assert len(crushed) == 10
-
- def test_full_crush_pipeline_number_array_types(self) -> None:
- """End-to-end: crushing a JSON number array via the public API."""
- crusher = _make_crusher(max_items=10)
- content = json.dumps(list(range(50)))
- result, was_modified, info = crusher._smart_crush_content(content)
-
- if was_modified:
- parsed = json.loads(result)
- assert isinstance(parsed, list)
- for item in parsed:
- assert isinstance(item, int | float), (
- f"Public API returned non-numeric item {item!r} in number array"
- )
-
+# Bug #1 (number array schema preservation) — invariant pinned by the
+# Rust port (`crates/headroom-core/src/transforms/smart_crusher/crushers.rs::
+# crush_number_array` + its unit tests) and the parity fixtures
+# (`tests/parity/fixtures/smart_crusher/number_array_40_changepoint*`).
+# The Python `_crush_number_array` helper that the previous tests
+# probed was removed when the Python implementation was retired in
+# Stage 3c.1b.
# ---------------------------------------------------------------------------
# Bug 2: Race condition on _current_field_semantics
@@ -212,79 +124,12 @@ class TestRecursionDepthLimit:
assert isinstance(parsed, list)
-# ---------------------------------------------------------------------------
-# Stage 3c.1 lockstep bug fixes (#1 percentile, #2 sequential, #3 rare-status,
-# #4 k-split). Each test pins the Python behavior post-fix; Rust has matching
-# tests so parity fixtures byte-equal both languages.
-# ---------------------------------------------------------------------------
-
-
-class TestStage3c1BugFixes:
- """Bugs fixed in lockstep with the Rust port at Stage 3c.1."""
-
- # Bug #1 — percentile off-by-one (cosmetic, strategy string only).
- def test_bug1_percentile_uses_linear_interpolation(self) -> None:
- from headroom.transforms.smart_crusher import _percentile_linear
-
- # n=10 [10..100]: p25 index = 0.25 * 9 = 2.25 → 30*0.75 + 40*0.25 = 32.5
- sorted_vals = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0]
- assert _percentile_linear(sorted_vals, 0.25) == 32.5
- assert _percentile_linear(sorted_vals, 0.75) == 77.5
- # n=1 → return the single value.
- assert _percentile_linear([42.0], 0.25) == 42.0
- # n=0 → 0.0 (defensive).
- assert _percentile_linear([], 0.25) == 0.0
-
- # Bug #2 — zero-padded string IDs misclassified as sequential.
- def test_bug2_zero_padded_strings_not_sequential(self) -> None:
- from headroom.transforms.smart_crusher import _detect_sequential_pattern
-
- # Pre-fix: int("001") → 1, so ["001",...,"005"] looked like 1..5
- # and was classified as sequential. Post-fix: had_non_string_numeric
- # stays False because every value came from a string → return False.
- zero_padded = ["001", "002", "003", "004", "005"]
- assert _detect_sequential_pattern(zero_padded, check_order=False) is False
-
- # Real ints (not strings) still classify as sequential.
- real_ints = [1, 2, 3, 4, 5]
- assert _detect_sequential_pattern(real_ints, check_order=False) is True
-
- # Bug #3 — rare-status detection cardinality cap.
- def test_bug3_high_cardinality_pareto(self) -> None:
- from headroom.transforms.smart_crusher import _detect_rare_status_values
-
- # 60×INFO + 25×WARN + 15 distinct error codes (cardinality 17).
- # Pre-fix: 17 > 10 → field skipped → 0 outliers.
- # Post-fix: top-2 covers 85% (60+25=85), K=2 ≤ 5, the 15 rare codes flagged.
- items = []
- for _ in range(60):
- items.append({"code": "INFO"})
- for _ in range(25):
- items.append({"code": "WARN"})
- for i in range(15):
- items.append({"code": f"ERR_{i}"})
- outliers = _detect_rare_status_values(items, common_fields={"code"})
- assert len(outliers) == 15
-
- def test_bug3_uniform_distribution_no_outliers(self) -> None:
- # 50 distinct values, 1 each → top-K never reaches 80% with K<=5.
- # Field correctly identified as non-categorical.
- from headroom.transforms.smart_crusher import _detect_rare_status_values
-
- items = [{"code": f"CAT_{i}"} for i in range(50)]
- assert _detect_rare_status_values(items, common_fields={"code"}) == []
-
- # Bug #4 — k-split overshoot when k_total=1.
- def test_bug4_k_split_no_overshoot_when_k_total_one(self) -> None:
- from headroom import SmartCrusherConfig
- from headroom.transforms.smart_crusher import SmartCrusher
-
- config = SmartCrusherConfig(min_items_to_analyze=1)
- crusher = SmartCrusher(config=config)
- # Force k_total=1 by passing a single-item list — the n<=8 fast
- # path returns n=1, so k_total=1.
- k_total, k_first, k_last, k_importance = crusher._compute_k_split(["only"], 1.0)
- assert k_total == 1
- assert k_first + k_last <= k_total, (
- f"BUG #4: k_first={k_first} + k_last={k_last} must not exceed k_total={k_total}"
- )
+# Stage 3c.1 lockstep bug-fix tests previously lived here; they probed
+# Python helpers (`_percentile_linear`, `_detect_sequential_pattern`,
+# `_detect_rare_status_values`, `_compute_k_split`) that were removed
+# along with the Python implementation in Stage 3c.1b. The Rust port
+# pins the same invariants — see the `bug1_*` / `bug2_*` / `bug3_*` /
+# `bug4_*` tests in `crates/headroom-core/src/transforms/smart_crusher/`
+# (notably `crushers.rs` and `analyzer.rs`). Parity fixtures
+# (`tests/parity/fixtures/smart_crusher/`) byte-compare the post-fix
+# behavior across the language boundary.
diff --git a/tests/test_transforms/test_universal_json_crush.py b/tests/test_transforms/test_universal_json_crush.py
deleted file mode 100644
index 4d6eb16fa..000000000
--- a/tests/test_transforms/test_universal_json_crush.py
+++ /dev/null
@@ -1,489 +0,0 @@
-"""Tests for universal JSON compression (all array types).
-
-Verifies that SmartCrusher handles arrays of dicts, strings, numbers,
-mixed types, and nested arrays — with consistent safety guarantees
-across all types.
-"""
-
-from __future__ import annotations
-
-import json
-
-import pytest
-
-from headroom.transforms.smart_crusher import (
- ArrayType,
- SmartCrusher,
- SmartCrusherConfig,
- _classify_array,
-)
-
-# =====================================================================
-# Fixtures
-# =====================================================================
-
-
-def crusher():
- """SmartCrusher configured for testing."""
- return SmartCrusher(
- config=SmartCrusherConfig(
- min_items_to_analyze=5,
- min_tokens_to_crush=0, # Always crush
- max_items_after_crush=15,
- )
- )
-
-
-def crusher_large_k():
- """SmartCrusher with higher max items for larger test arrays."""
- return SmartCrusher(
- config=SmartCrusherConfig(
- min_items_to_analyze=5,
- min_tokens_to_crush=0,
- max_items_after_crush=50,
- )
- )
-
-
-# =====================================================================
-# Type Classification
-# =====================================================================
-
-
-class TestClassifyArray:
- def test_dict_array(self):
- assert _classify_array([{"a": 1}, {"b": 2}]) == ArrayType.DICT_ARRAY
-
- def test_string_array(self):
- assert _classify_array(["hello", "world", "foo"]) == ArrayType.STRING_ARRAY
-
- def test_number_array_int(self):
- assert _classify_array([1, 2, 3]) == ArrayType.NUMBER_ARRAY
-
- def test_number_array_float(self):
- assert _classify_array([1.0, 2.5, 3.7]) == ArrayType.NUMBER_ARRAY
-
- def test_number_array_mixed_int_float(self):
- assert _classify_array([1, 2.5, 3]) == ArrayType.NUMBER_ARRAY
-
- def test_bool_array(self):
- assert _classify_array([True, False, True]) == ArrayType.BOOL_ARRAY
-
- def test_nested_array(self):
- assert _classify_array([[1, 2], [3, 4]]) == ArrayType.NESTED_ARRAY
-
- def test_mixed_array(self):
- assert _classify_array([{"a": 1}, "string", 42]) == ArrayType.MIXED_ARRAY
-
- def test_empty(self):
- assert _classify_array([]) == ArrayType.EMPTY
-
- def test_single_dict(self):
- assert _classify_array([{"key": "val"}]) == ArrayType.DICT_ARRAY
-
- def test_single_string(self):
- assert _classify_array(["only"]) == ArrayType.STRING_ARRAY
-
- def test_none_values(self):
- # Arrays with None mixed in are MIXED
- assert _classify_array([1, None, 3]) == ArrayType.MIXED_ARRAY
-
- def test_bool_not_confused_with_int(self):
- # Python's True/False are int subclasses — we handle this
- assert _classify_array([True, False]) == ArrayType.BOOL_ARRAY
- # But mixed bools and ints should be MIXED or NUMBER depending on impl
- result = _classify_array([True, 42])
- assert result in (ArrayType.MIXED_ARRAY, ArrayType.NUMBER_ARRAY)
-
-
-# =====================================================================
-# String Array Compression
-# =====================================================================
-
-
-class TestCrushStringArray:
- def test_basic_compression(self, crusher):
- strings = [f"item_{i}" for i in range(100)]
- crushed, strategy = crusher._crush_string_array(strings)
- assert len(crushed) < len(strings)
- assert "string:adaptive" in strategy
-
- def test_errors_always_preserved(self, crusher):
- strings = ["ok"] * 50 + ["error: connection timeout", "failed: auth denied"] + ["ok"] * 48
- crushed, strategy = crusher._crush_string_array(strings)
- assert any("error" in s for s in crushed)
- assert any("failed" in s for s in crushed)
-
- def test_first_last_kept(self, crusher):
- strings = [f"item_{i}" for i in range(50)]
- crushed, strategy = crusher._crush_string_array(strings)
- # First item always present
- assert strings[0] in crushed
- # Last item always present
- assert strings[-1] in crushed
-
- def test_dedup_reduces_output(self, crusher):
- # 95 identical + 5 unique
- strings = ["repeated_value"] * 95 + [f"unique_{i}" for i in range(5)]
- crushed, strategy = crusher._crush_string_array(strings)
- # Should massively reduce — not keep 95 copies
- assert len(crushed) < 20
- # All 5 unique values should survive (they have high info value)
- for i in range(5):
- assert f"unique_{i}" in crushed
-
- def test_below_threshold_passthrough(self, crusher):
- strings = ["a", "b", "c"] # Below min_items_to_analyze=5
- # Direct method call — should passthrough since <= 8
- crushed, strategy = crusher._crush_string_array(strings)
- assert crushed == strings
- assert "passthrough" in strategy
-
- def test_empty_strings_handled(self, crusher):
- strings = [""] * 20
- crushed, strategy = crusher._crush_string_array(strings)
- # Should not crash
- assert isinstance(crushed, list)
-
- def test_unicode_strings(self, crusher):
- strings = [f"日本語テスト_{i}" for i in range(50)]
- crushed, strategy = crusher._crush_string_array(strings)
- assert len(crushed) < len(strings)
- assert all(isinstance(s, str) for s in crushed)
-
- def test_length_anomalies_preserved(self, crusher_large_k):
- # Most strings are short, one is very long
- strings = ["short"] * 95 + ["x" * 10000] + ["short"] * 4
- crushed, strategy = crusher_large_k._crush_string_array(strings)
- assert any(len(s) > 1000 for s in crushed)
-
-
-# =====================================================================
-# Number Array Compression
-# =====================================================================
-
-
-class TestCrushNumberArray:
- def test_basic_compression(self, crusher):
- numbers = [42.0 + i * 0.1 for i in range(100)]
- crushed, strategy = crusher._crush_number_array(numbers)
- assert len(crushed) < len(numbers)
- assert "number:adaptive" in strategy
-
- def test_stats_in_strategy_not_array(self, crusher):
- numbers = list(range(100))
- crushed, strategy = crusher._crush_number_array(numbers)
- # Stats should be in the strategy string, not in the array
- assert "min=" in strategy
- assert "max=" in strategy
- # Array should contain only numbers (schema-preserving)
- for item in crushed:
- assert isinstance(item, int | float)
-
- def test_outliers_preserved(self, crusher):
- # Normal values around 50 with one extreme outlier
- numbers = [50.0 + i * 0.01 for i in range(100)] + [999.9]
- crushed, strategy = crusher._crush_number_array(numbers)
- assert 999.9 in crushed
- assert "outliers" in strategy
-
- def test_all_identical(self, crusher):
- numbers = [42.0] * 100
- crushed, strategy = crusher._crush_number_array(numbers)
- # With all identical, should compress heavily
- # Summary + a few representatives
- numeric_values = [v for v in crushed if isinstance(v, int | float)]
- assert all(v == 42.0 for v in numeric_values)
-
- def test_first_last_kept(self, crusher):
- numbers = list(range(50))
- crushed, strategy = crusher._crush_number_array(numbers)
- numeric_values = [v for v in crushed if isinstance(v, int | float)]
- assert 0 in numeric_values # First
- assert 49 in numeric_values # Last
-
- def test_change_point_preserved(self, crusher_large_k):
- # Stable at 10, then jumps to 100
- numbers = [10.0] * 50 + [100.0] * 50
- crushed, strategy = crusher_large_k._crush_number_array(numbers)
- numeric_values = [v for v in crushed if isinstance(v, int | float)]
- # Both 10.0 and 100.0 should be present
- assert 10.0 in numeric_values
- assert 100.0 in numeric_values
-
- def test_nan_inf_filtered(self, crusher):
- numbers = [1.0, 2.0, float("nan"), float("inf"), 3.0] * 10
- crushed, strategy = crusher._crush_number_array(numbers)
- # Should not crash; stats in strategy based on finite values
- assert "min=" in strategy
- assert "max=" in strategy
-
- def test_integers_preserved_as_int(self, crusher):
- numbers = list(range(50))
- crushed, strategy = crusher._crush_number_array(numbers)
- numeric_values = [v for v in crushed if isinstance(v, int | float)]
- # Integers should remain integers (not converted to float)
- assert any(isinstance(v, int) for v in numeric_values)
-
- def test_statistics_accuracy(self, crusher):
- numbers = list(range(1, 101)) # 1 to 100
- crushed, strategy = crusher._crush_number_array(numbers)
- # Stats are in the strategy string
- assert "min=1" in strategy
- assert "max=100" in strategy
- assert "mean=50.5" in strategy
-
-
-# =====================================================================
-# Mixed Array Compression
-# =====================================================================
-
-
-class TestCrushMixedArray:
- def test_basic_compression(self, crusher_large_k):
- mixed = [{"id": i} for i in range(30)] + [f"msg_{i}" for i in range(30)]
- crushed, strategy = crusher_large_k._crush_mixed_array(mixed)
- # With diversity-aware K, unique items may all be kept.
- # Verify compression happened OR items are preserved due to high diversity.
- assert len(crushed) <= len(mixed)
- assert "mixed" in strategy
-
- def test_small_groups_kept(self, crusher):
- # 50 dicts + 3 strings (below threshold)
- mixed = [{"id": i} for i in range(50)] + ["rare_1", "rare_2", "rare_3"]
- crushed, strategy = crusher._crush_mixed_array(mixed)
- # All 3 rare strings should be kept (below min_items threshold)
- assert "rare_1" in crushed
- assert "rare_2" in crushed
- assert "rare_3" in crushed
-
- def test_errors_across_types(self, crusher_large_k):
- mixed = (
- [{"status": "ok"}] * 30
- + [{"status": "error: timeout"}]
- + ["error: auth failed"]
- + [f"ok_{i}" for i in range(30)]
- )
- crushed, strategy = crusher_large_k._crush_mixed_array(mixed)
- crushed_str = json.dumps(crushed)
- assert "error: timeout" in crushed_str
- assert "error: auth failed" in crushed_str
-
- def test_original_order_preserved(self, crusher_large_k):
- mixed = [{"id": i} for i in range(20)] + [f"str_{i}" for i in range(20)]
- crushed, strategy = crusher_large_k._crush_mixed_array(mixed)
- # Dicts should come before strings (original order)
- first_str_idx = next(
- (i for i, item in enumerate(crushed) if isinstance(item, str)), len(crushed)
- )
- last_dict_idx = max(
- (i for i, item in enumerate(crushed) if isinstance(item, dict)), default=-1
- )
- assert last_dict_idx < first_str_idx
-
- def test_passthrough_small(self, crusher):
- mixed = [1, "two", {"three": 3}]
- crushed, strategy = crusher._crush_mixed_array(mixed)
- assert crushed == mixed
- assert "passthrough" in strategy
-
-
-# =====================================================================
-# Adaptive K
-# =====================================================================
-
-
-class TestAdaptiveK:
- def test_scales_with_n(self, crusher):
- """K grows sublinearly with collection size (or saturates at max_items)."""
- small = [f"item_{i}" for i in range(20)]
- large = [f"item_{i}" for i in range(500)]
-
- k_small = crusher._compute_k_split(small)[0]
- k_large = crusher._compute_k_split(large)[0]
-
- # With diversity-aware K, both may saturate at max_items_after_crush
- # for highly unique items. The key property: k never exceeds max_items.
- assert k_large >= k_small
- assert k_large <= crusher.config.max_items_after_crush
-
- def test_respects_max_items(self, crusher):
- items = [f"item_{i}" for i in range(1000)]
- k_total, _, _, _ = crusher._compute_k_split(items)
- assert k_total <= crusher.config.max_items_after_crush
-
- def test_first_last_fractions(self, crusher):
- items = [f"item_{i}" for i in range(100)]
- k_total, k_first, k_last, k_importance = crusher._compute_k_split(items)
- # First and last should be roughly the configured fractions
- assert k_first >= 1
- assert k_last >= 1
- assert k_first + k_last + k_importance == k_total
-
- def test_homogeneous_vs_diverse(self, crusher_large_k):
- """Homogeneous data should produce smaller K than diverse data."""
- homogeneous = ["same value"] * 100
- diverse = [f"unique_value_{i}_{'x' * (i * 10)}" for i in range(100)]
-
- k_homo = crusher_large_k._compute_k_split(homogeneous)[0]
- k_diverse = crusher_large_k._compute_k_split(diverse)[0]
-
- # Diverse should need more items (or at least equal)
- assert k_diverse >= k_homo
-
-
-# =====================================================================
-# Safety Guarantees (parametrized across types)
-# =====================================================================
-
-
-class TestSafetyGuarantees:
- @pytest.mark.parametrize(
- "items,item_type",
- [
- ([{"status": "ok"}] * 50 + [{"status": "error: timeout"}], "dict"),
- (["ok"] * 50 + ["error: connection failed"], "string"),
- (
- [{"id": i} for i in range(30)]
- + ["error: auth denied"]
- + [f"ok_{i}" for i in range(30)],
- "mixed",
- ),
- ],
- ids=["dict_array", "string_array", "mixed_array"],
- )
- def test_errors_never_dropped(self, crusher_large_k, items, item_type):
- """Error items must be preserved regardless of array type."""
- if item_type == "dict":
- crushed, _ = crusher_large_k._crush_string_array([json.dumps(i) for i in items])
- crushed_text = " ".join(crushed)
- elif item_type == "string":
- crushed, _ = crusher_large_k._crush_string_array(items)
- crushed_text = " ".join(crushed)
- elif item_type == "mixed":
- crushed, _ = crusher_large_k._crush_mixed_array(items)
- crushed_text = json.dumps(crushed)
-
- assert "error" in crushed_text.lower()
-
- @pytest.mark.parametrize(
- "items",
- [
- [f"item_{i}" for i in range(50)],
- list(range(50)),
- ],
- ids=["string_array", "number_array"],
- )
- def test_first_last_always_present(self, crusher, items):
- """First and last items must be present in output."""
- if isinstance(items[0], str):
- crushed, _ = crusher._crush_string_array(items)
- assert items[0] in crushed
- assert items[-1] in crushed
- else:
- crushed, _ = crusher._crush_number_array(items)
- numeric = [v for v in crushed if isinstance(v, int | float)]
- assert items[0] in numeric
- assert items[-1] in numeric
-
- @pytest.mark.parametrize(
- "items",
- [
- ["a", "b", "c"],
- [1, 2, 3],
- [{"x": 1}, "y", 2],
- ],
- ids=["string_below", "number_below", "mixed_below"],
- )
- def test_passthrough_below_min_items(self, crusher, items):
- """Arrays below min_items_to_analyze pass through unchanged."""
- if all(isinstance(i, str) for i in items):
- crushed, strategy = crusher._crush_string_array(items)
- elif all(isinstance(i, int | float) for i in items):
- crushed, strategy = crusher._crush_number_array(items)
- else:
- crushed, strategy = crusher._crush_mixed_array(items)
- assert "passthrough" in strategy
-
-
-# =====================================================================
-# Integration: Full pipeline
-# =====================================================================
-
-
-class TestFullPipelineIntegration:
- """Test that new types work through the compress() API."""
-
- def test_string_array_via_compress(self):
- from headroom import compress
-
- strings = [f"log line {i}: GET /api 200" for i in range(100)]
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "user", "content": "Show logs"},
- {
- "role": "assistant",
- "content": None,
- "tool_calls": [
- {
- "id": "call_1",
- "type": "function",
- "function": {"name": "get_logs", "arguments": "{}"},
- }
- ],
- },
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(strings)},
- ]
- result = compress(messages)
- assert result.tokens_saved > 0
- assert result.compression_ratio > 0
-
- def test_number_array_via_compress(self):
- from headroom import compress
-
- numbers = [42.0 + i * 0.1 for i in range(200)]
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "user", "content": "Show metrics"},
- {
- "role": "assistant",
- "content": None,
- "tool_calls": [
- {
- "id": "call_1",
- "type": "function",
- "function": {"name": "get_metrics", "arguments": "{}"},
- }
- ],
- },
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(numbers)},
- ]
- result = compress(messages)
- assert result.tokens_saved > 0
-
- def test_dict_array_unchanged(self):
- """Verify dict arrays still work (regression test)."""
- from headroom import compress
-
- data = [{"id": i, "name": f"user_{i}", "status": "active"} for i in range(100)]
- messages = [
- {"role": "system", "content": "You are helpful."},
- {"role": "user", "content": "List users"},
- {
- "role": "assistant",
- "content": None,
- "tool_calls": [
- {
- "id": "call_1",
- "type": "function",
- "function": {"name": "list_users", "arguments": "{}"},
- }
- ],
- },
- {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(data)},
- ]
- result = compress(messages)
- assert result.tokens_saved > 0
- assert result.compression_ratio > 0