Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

test(toin): delete the permanently skipped PR-B5 get_recommendation tests (#4090)

Tejas Chopra85c35799ea1e
diff --git a/tests/test_critical_gaps.py b/tests/test_critical_gaps.py
index 0ec6fb6cc..9153a99f7 100644
--- a/tests/test_critical_gaps.py
+++ b/tests/test_critical_gaps.py
@@ -845,12 +845,6 @@ class TestTOINHighPriorityFixes:
# (auto-save happens inside record_compression)
assert not toin._dirty
- @pytest.mark.skip(
- reason="PR-B5: get_recommendation retired; preserve_fields lives on the aggregated ToolPattern instead"
- )
- def test_toin_preserves_fields_returns_list(self):
- """Retired in PR-B5 along with the request-time hint API."""
-
class TestCompressionStoreHighPriorityFixes:
"""Additional HIGH priority tests for CompressionStore."""
diff --git a/tests/test_toin.py b/tests/test_toin.py
index 40205e339..acacc7e20 100644
--- a/tests/test_toin.py
+++ b/tests/test_toin.py
@@ -1,9 +1,8 @@
"""Tests for Tool Output Intelligence Network (TOIN).
-PR-B5 retired the request-time hint API. Tests that exercised the old
-`get_recommendation()` / `CompressionHint` shape are skipped at module
-level — the new observation-only contract is covered by
-`tests/test_toin_observation_only.py` and `tests/test_toin_publish.py`.
+PR-B5 retired the request-time hint API. The observation-only contract
+is covered by `tests/test_toin_observation_only.py` and
+`tests/test_toin_publish.py`.
"""
import os
@@ -331,47 +330,6 @@ class TestToolIntelligenceNetwork:
# Field should be in commonly_retrieved_fields after 3+ retrievals
assert len(pattern.commonly_retrieved_fields) > 0
- # PR-B5: the following tests exercised the request-time hint API
- # that's now retired. They're skipped wholesale; the new contract
- # ("get_recommendation always returns None and emits a deprecation
- # warning") is covered by tests/test_toin_observation_only.py.
-
- @pytest.mark.skip(
- reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
- )
- def test_get_recommendation_no_data(self):
- pass
-
- @pytest.mark.skip(
- reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
- )
- def test_get_recommendation_insufficient_samples(self):
- pass
-
- @pytest.mark.skip(
- reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
- )
- def test_get_recommendation_aggressive_compression(self):
- pass
-
- @pytest.mark.skip(
- reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
- )
- def test_get_recommendation_conservative_compression(self):
- pass
-
- @pytest.mark.skip(
- reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
- )
- def test_get_recommendation_skip_compression(self):
- pass
-
- @pytest.mark.skip(
- reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
- )
- def test_get_recommendation_disabled(self):
- pass
-
def test_get_stats(self):
"""get_stats returns overall statistics."""
toin = ToolIntelligenceNetwork()
diff --git a/tests/test_toin_feedback.py b/tests/test_toin_feedback.py
index 46ad62a92..7ff35d5ab 100644
--- a/tests/test_toin_feedback.py
+++ b/tests/test_toin_feedback.py
@@ -10,7 +10,6 @@ from headroom.cache.compression_store import get_compression_store, reset_compre
from headroom.telemetry import (
TOINConfig,
ToolIntelligenceNetwork,
- ToolPattern,
ToolSignature,
get_toin,
reset_toin,
@@ -81,94 +80,6 @@ def test_kompress_ccr_retrieval_updates_toin():
assert stats["total_retrievals"] == 1
[email protected](reason="PR-B5: observations counter and request-time hint API retired")
-class TestGetRecommendationObservations:
- """Bug 1: get_recommendation() should increment observations counter."""
-
- def test_increments_observations_when_pattern_exists(self):
- """get_recommendation() should increment observations when pattern exists."""
- config = _make_config(min_samples=5)
- toin = ToolIntelligenceNetwork(config=config)
- sig = _make_signature("obs_test_hash")
-
- # Record enough compressions to create a pattern with sufficient samples
- for _ in range(15):
- toin.record_compression(
- tool_signature=sig,
- original_count=100,
- compressed_count=20,
- original_tokens=5000,
- compressed_tokens=1000,
- strategy="top_n",
- )
-
- # Get recommendation
- toin.get_recommendation(sig)
-
- # Check observations incremented
- pattern = toin._patterns[("global", "unknown", "unknown", sig.structure_hash)]
- assert pattern.observations == 1
-
- # Call again
- toin.get_recommendation(sig)
- assert pattern.observations == 2
-
- def test_increments_observations_even_below_min_samples(self):
- """observations increments even when sample_size < min_samples."""
- config = _make_config(min_samples=100)
- toin = ToolIntelligenceNetwork(config=config)
- sig = _make_signature("low_sample_hash")
-
- # Record just a few compressions (below min_samples)
- for _ in range(3):
- toin.record_compression(
- tool_signature=sig,
- original_count=50,
- compressed_count=10,
- original_tokens=2000,
- compressed_tokens=500,
- strategy="top_n",
- )
-
- result = toin.get_recommendation(sig)
- assert result.source == "local" # Not enough samples
-
- pattern = toin._patterns[("global", "unknown", "unknown", sig.structure_hash)]
- assert pattern.observations == 1
-
- def test_no_increment_for_unknown_pattern(self):
- """get_recommendation() should NOT increment for unknown patterns."""
- config = _make_config()
- toin = ToolIntelligenceNetwork(config=config)
- sig = _make_signature("nonexistent_hash")
-
- result = toin.get_recommendation(sig)
- assert result.source == "default"
- assert result.reason == "No pattern data for this tool type"
-
- # No pattern exists, nothing to increment
- assert "nonexistent_hash" not in toin._patterns
-
- def test_observations_survives_serialization(self):
- """observations field should serialize and deserialize correctly."""
- pattern = ToolPattern(
- tool_signature_hash="serial_test",
- total_compressions=10,
- observations=42,
- )
-
- d = pattern.to_dict()
- assert d["observations"] == 42
-
- restored = ToolPattern.from_dict(d)
- assert restored.observations == 42
-
- def test_observations_defaults_to_zero(self):
- """observations defaults to 0 for new patterns."""
- pattern = ToolPattern(tool_signature_hash="new_pattern")
- assert pattern.observations == 0
-
-
class TestRecordRetrievalPopulatesFields:
"""Bug 1 related: record_retrieval with query_fields populates field data."""
diff --git a/tests/test_toin_fixes.py b/tests/test_toin_fixes.py
index cb32332b8..b463e29a0 100644
--- a/tests/test_toin_fixes.py
+++ b/tests/test_toin_fixes.py
@@ -68,99 +68,6 @@ def fresh_store():
reset_compression_store()
- reason="PR-B5: strategy-recommendation API retired (get_recommendation returns None)"
-)
-class TestStrategySuccessRates:
- """Test that strategy_success_rates are used in recommendations."""
-
- def test_recommends_strategy_with_high_success_rate(self, fresh_toin):
- """Strategy with success rate >= 0.5 should be recommended."""
- items = [{"id": i, "score": 100 - i} for i in range(20)]
- signature = ToolSignature.from_items(items)
-
- # Record compressions to build pattern
- for _ in range(10):
- fresh_toin.record_compression(
- tool_signature=signature,
- original_count=20,
- compressed_count=10,
- original_tokens=2000,
- compressed_tokens=1000,
- strategy="smart_sample",
- )
-
- # Set high success rate
- pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
- pattern.strategy_success_rates["smart_sample"] = 0.8
- pattern.optimal_strategy = "smart_sample"
-
- # Get recommendation
- hint = fresh_toin.get_recommendation(signature, "test query")
-
- assert hint.recommended_strategy == "smart_sample"
-
- def test_rejects_strategy_with_low_success_rate(self, fresh_toin):
- """Strategy with success rate < 0.5 should NOT be recommended."""
- items = [{"id": i, "score": 100 - i} for i in range(20)]
- signature = ToolSignature.from_items(items)
-
- # Record compressions
- for _ in range(10):
- fresh_toin.record_compression(
- tool_signature=signature,
- original_count=20,
- compressed_count=10,
- original_tokens=2000,
- compressed_tokens=1000,
- strategy="bad_strategy",
- )
-
- # Set low success rate
- pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
- pattern.strategy_success_rates["bad_strategy"] = 0.2
- pattern.optimal_strategy = "bad_strategy"
-
- # Get recommendation
- hint = fresh_toin.get_recommendation(signature, "test query")
-
- # Should not recommend the bad strategy
- assert hint.recommended_strategy != "bad_strategy"
- # Confidence should be reduced
- assert "low success" in hint.reason.lower()
-
- def test_finds_best_strategy_when_optimal_is_bad(self, fresh_toin):
- """When optimal_strategy has low success, find a better alternative."""
- items = [{"id": i, "score": 100 - i} for i in range(20)]
- signature = ToolSignature.from_items(items)
-
- # Record compressions
- for _ in range(10):
- fresh_toin.record_compression(
- tool_signature=signature,
- original_count=20,
- compressed_count=10,
- original_tokens=2000,
- compressed_tokens=1000,
- strategy="smart_sample",
- )
-
- # Set up multiple strategies with different success rates
- pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
- pattern.strategy_success_rates = {
- "bad_strategy": 0.2,
- "good_strategy": 0.9,
- }
- pattern.optimal_strategy = "bad_strategy"
-
- # Get recommendation
- hint = fresh_toin.get_recommendation(signature, "test query")
-
- # Should recommend the better strategy
- assert hint.recommended_strategy == "good_strategy"
- assert "using good_strategy instead" in hint.reason
-
-
class TestPreserveFieldsMerging:
"""Test preserve_fields merging in federated learning."""
@@ -352,166 +259,6 @@ class TestUserCountTracking:
assert pattern.user_count >= 3
- reason="PR-B5: get_recommendation retired; field-weighting now consumed only by toin publish"
-)
-class TestFieldRetrievalFrequencyWeighting:
- """Test field_retrieval_frequency weighting in preserve_fields."""
-
- def test_query_fields_prioritized_in_preserve_fields(self, fresh_toin):
- """Fields mentioned in query should be prioritized."""
- items = [{"id": i, "status": "ok", "category": f"cat_{i}"} for i in range(20)]
- signature = ToolSignature.from_items(items)
-
- # Build pattern with field retrieval data
- for _ in range(10):
- fresh_toin.record_compression(
- tool_signature=signature,
- original_count=20,
- compressed_count=10,
- original_tokens=2000,
- compressed_tokens=1000,
- strategy="smart_sample",
- )
-
- # Record retrievals for "status" field
- status_hash = fresh_toin._hash_field_name("status")
- pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
- pattern.field_retrieval_frequency = {
- status_hash: 50,
- fresh_toin._hash_field_name("category"): 10,
- }
- pattern.preserve_fields = [status_hash]
-
- # Get recommendation with query mentioning "status"
- hint = fresh_toin.get_recommendation(signature, "status:error")
-
- # status hash should be in preserve_fields
- assert status_hash in hint.preserve_fields
-
- def test_preserve_fields_sorted_by_frequency(self, fresh_toin):
- """preserve_fields should be sorted by retrieval frequency."""
- items = [{"id": i} for i in range(20)]
- signature = ToolSignature.from_items(items)
-
- # Build pattern
- for _ in range(10):
- fresh_toin.record_compression(
- tool_signature=signature,
- original_count=20,
- compressed_count=10,
- original_tokens=2000,
- compressed_tokens=1000,
- strategy="smart_sample",
- )
-
- pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
- field_a = fresh_toin._hash_field_name("field_a")
- field_b = fresh_toin._hash_field_name("field_b")
- field_c = fresh_toin._hash_field_name("field_c")
-
- pattern.field_retrieval_frequency = {
- field_a: 10,
- field_b: 50, # Most frequent
- field_c: 30,
- }
- pattern.preserve_fields = [field_a, field_b, field_c]
-
- # Get recommendation (no query context)
- hint = fresh_toin.get_recommendation(signature, "")
-
- # Should be sorted by frequency
- if len(hint.preserve_fields) >= 3:
- # field_b should come before field_c which should come before field_a
- b_idx = hint.preserve_fields.index(field_b) if field_b in hint.preserve_fields else -1
- c_idx = hint.preserve_fields.index(field_c) if field_c in hint.preserve_fields else -1
- hint.preserve_fields.index(field_a) if field_a in hint.preserve_fields else -1
-
- if b_idx >= 0 and c_idx >= 0:
- assert b_idx < c_idx, "Higher frequency field should come first"
-
-
[email protected](reason="PR-B5: get_recommendation retired (returns None / DeprecationWarning)")
-class TestQueryContextUsage:
- """Test query_context usage in recommendations."""
-
- def test_exhaustive_query_keywords_detected(self, fresh_toin):
- """Exhaustive query keywords should trigger conservative compression."""
- items = [{"id": i, "score": 100 - i} for i in range(50)]
- signature = ToolSignature.from_items(items)
-
- # Build pattern with aggressive compression normally
- for _ in range(10):
- fresh_toin.record_compression(
- tool_signature=signature,
- original_count=50,
- compressed_count=10,
- original_tokens=5000,
- compressed_tokens=1000,
- strategy="smart_sample",
- )
-
- # Low retrieval rate = aggressive compression
- pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
- pattern.total_retrievals = 0
-
- # Query with exhaustive keyword
- hint = fresh_toin.get_recommendation(signature, "list all items in category")
-
- # Should be more conservative
- assert hint.max_items >= 40
- assert "exhaustive query" in hint.reason.lower()
- assert hint.compression_level == "conservative"
-
- def test_every_keyword_triggers_conservative(self, fresh_toin):
- """'every' keyword should trigger conservative compression."""
- items = [{"id": i} for i in range(50)]
- signature = ToolSignature.from_items(items)
-
- for _ in range(10):
- fresh_toin.record_compression(
- tool_signature=signature,
- original_count=50,
- compressed_count=10,
- original_tokens=5000,
- compressed_tokens=1000,
- strategy="smart_sample",
- )
-
- pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
- pattern.total_retrievals = 0
-
- hint = fresh_toin.get_recommendation(signature, "find every user")
-
- assert "exhaustive query" in hint.reason.lower()
-
- def test_partial_pattern_matching(self, fresh_toin):
- """Partial pattern matching should boost max_items."""
- items = [{"id": i, "status": "ok"} for i in range(50)]
- signature = ToolSignature.from_items(items)
-
- for _ in range(10):
- fresh_toin.record_compression(
- tool_signature=signature,
- original_count=50,
- compressed_count=10,
- original_tokens=5000,
- compressed_tokens=1000,
- strategy="smart_sample",
- )
-
- pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
- pattern.total_retrievals = 0
- # Add a problematic query pattern
- pattern.common_query_patterns = ["status:*"]
-
- # Query that uses the same field
- hint = fresh_toin.get_recommendation(signature, "status:error")
-
- # Should match the pattern
- assert hint.max_items >= 25 or "retrieval pattern" in hint.reason
-
-
class TestFeedbackStrategyTracking:
"""Test strategy tracking in compression feedback."""
diff --git a/tests/test_toin_full_integration.py b/tests/test_toin_full_integration.py
index a74086a54..4d56f2d92 100644
--- a/tests/test_toin_full_integration.py
+++ b/tests/test_toin_full_integration.py
@@ -247,208 +247,6 @@ class TestTOINPersistenceAcrossInstances:
print("\n[PASS] TOIN persistence works correctly")
- reason="PR-B5: get_recommendation retired; feedback-loop covered by test_toin_observation_only.py"
-)
-class TestTOINFullFeedbackLoop:
- """Test 3: Verify TOIN feedback loop with recommendations."""
-
- def test_toin_full_feedback_loop(self, sample_tool_signature):
- """Verify TOIN learns from high retrieval rate and recommends skip."""
- print("\n" + "=" * 60)
- print("TEST: test_toin_full_feedback_loop")
- print("=" * 60)
-
- with tempfile.TemporaryDirectory() as tmpdir:
- storage_path = str(Path(tmpdir) / "toin_feedback_test.json")
- config = TOINConfig(
- storage_path=storage_path,
- min_samples_for_recommendation=5, # Lower threshold for test
- high_retrieval_threshold=0.5, # 50% retrieval = high
- )
- toin = ToolIntelligenceNetwork(config)
-
- print("\n--- Phase 1: Record compressions ---")
- # Record 5 compressions with same tool signature
- for i in range(5):
- toin.record_compression(
- tool_signature=sample_tool_signature,
- original_count=100,
- compressed_count=15,
- original_tokens=5000,
- compressed_tokens=750,
- strategy="smart_sample",
- )
- print(f" Recorded compression {i + 1}")
-
- print("\n--- Phase 2: Record retrievals (simulating high retrieval rate) ---")
- # Record 3 full retrievals (60% retrieval rate = high)
- for i in range(3):
- toin.record_retrieval(
- tool_signature_hash=sample_tool_signature.structure_hash,
- retrieval_type="full", # Full retrieval = compression too aggressive
- strategy="smart_sample",
- )
- print(f" Recorded full retrieval {i + 1}")
-
- # Get pattern stats
- pattern = toin.get_pattern(sample_tool_signature.structure_hash)
- print("\n--- Pattern Stats ---")
- print(f" total_compressions: {pattern.total_compressions}")
- print(f" total_retrievals: {pattern.total_retrievals}")
- print(f" retrieval_rate: {pattern.retrieval_rate:.1%}")
- print(f" full_retrieval_rate: {pattern.full_retrieval_rate:.1%}")
- print(f" skip_compression_recommended: {pattern.skip_compression_recommended}")
-
- # Get recommendation
- print("\n--- Getting Recommendation ---")
- hint = toin.get_recommendation(sample_tool_signature)
- print(f" source: {hint.source}")
- print(f" skip_compression: {hint.skip_compression}")
- print(f" compression_level: {hint.compression_level}")
- print(f" max_items: {hint.max_items}")
- print(f" confidence: {hint.confidence:.3f}")
- print(f" reason: {hint.reason}")
- print(f" based_on_samples: {hint.based_on_samples}")
-
- # Verify high retrieval rate triggers skip recommendation
- # With 60% retrieval rate (3/5) and full_retrieval_rate of 100% (3/3),
- # TOIN should recommend skipping compression
- retrieval_rate = pattern.retrieval_rate
- assert retrieval_rate >= 0.5, (
- f"Expected retrieval rate >= 50%, got {retrieval_rate:.1%}"
- )
-
- # With high retrieval rate and high full retrieval rate, should skip
- if pattern.full_retrieval_rate > 0.8:
- assert hint.skip_compression or hint.compression_level in (
- "none",
- "conservative",
- ), (
- f"High full retrieval rate should trigger skip or conservative, "
- f"got compression_level={hint.compression_level}"
- )
- print("\n[PASS] High retrieval rate correctly influences recommendation")
- else:
- print("\n[INFO] Full retrieval rate not high enough for skip recommendation")
- print(f" full_retrieval_rate: {pattern.full_retrieval_rate:.1%}")
-
- print("\n[PASS] TOIN feedback loop works correctly")
-
-
- reason="PR-B5: get_recommendation retired; confidence-progression validated via record + get_pattern instead"
-)
-class TestTOINProgressiveConfidence:
- """Test 4: Verify TOIN confidence increases with sample size."""
-
- def test_toin_progressive_confidence(self, sample_tool_signature):
- """Verify confidence increases with more samples."""
- print("\n" + "=" * 60)
- print("TEST: test_toin_progressive_confidence")
- print("=" * 60)
-
- with tempfile.TemporaryDirectory() as tmpdir:
- storage_path = str(Path(tmpdir) / "toin_confidence_test.json")
- config = TOINConfig(
- storage_path=storage_path,
- min_samples_for_recommendation=3,
- )
- toin = ToolIntelligenceNetwork(config)
-
- confidence_history = []
-
- # Batch 1: Record 1 compression
- print("\n--- Batch 1: 1 compression ---")
- toin.record_compression(
- tool_signature=sample_tool_signature,
- original_count=100,
- compressed_count=15,
- original_tokens=5000,
- compressed_tokens=750,
- strategy="smart_sample",
- )
- pattern = toin.get_pattern(sample_tool_signature.structure_hash)
- hint = toin.get_recommendation(sample_tool_signature)
- confidence_history.append(pattern.confidence)
- print(f" sample_size: {pattern.sample_size}")
- print(f" confidence: {pattern.confidence:.3f}")
- print(f" hint.source: {hint.source}")
-
- # Batch 2: Record 2 more compressions
- print("\n--- Batch 2: +2 compressions (total: 3) ---")
- for _ in range(2):
- toin.record_compression(
- tool_signature=sample_tool_signature,
- original_count=100,
- compressed_count=15,
- original_tokens=5000,
- compressed_tokens=750,
- strategy="smart_sample",
- )
- pattern = toin.get_pattern(sample_tool_signature.structure_hash)
- hint = toin.get_recommendation(sample_tool_signature)
- confidence_history.append(pattern.confidence)
- print(f" sample_size: {pattern.sample_size}")
- print(f" confidence: {pattern.confidence:.3f}")
- print(f" hint.source: {hint.source}")
-
- # Batch 3: Record 2 more compressions
- print("\n--- Batch 3: +2 compressions (total: 5) ---")
- for _ in range(2):
- toin.record_compression(
- tool_signature=sample_tool_signature,
- original_count=100,
- compressed_count=15,
- original_tokens=5000,
- compressed_tokens=750,
- strategy="smart_sample",
- )
- pattern = toin.get_pattern(sample_tool_signature.structure_hash)
- hint = toin.get_recommendation(sample_tool_signature)
- confidence_history.append(pattern.confidence)
- print(f" sample_size: {pattern.sample_size}")
- print(f" confidence: {pattern.confidence:.3f}")
- print(f" hint.source: {hint.source}")
-
- # Batch 4: Add many more to boost confidence
- print("\n--- Batch 4: +15 compressions (total: 20) ---")
- for _ in range(15):
- toin.record_compression(
- tool_signature=sample_tool_signature,
- original_count=100,
- compressed_count=15,
- original_tokens=5000,
- compressed_tokens=750,
- strategy="smart_sample",
- )
- pattern = toin.get_pattern(sample_tool_signature.structure_hash)
- hint = toin.get_recommendation(sample_tool_signature)
- confidence_history.append(pattern.confidence)
- print(f" sample_size: {pattern.sample_size}")
- print(f" confidence: {pattern.confidence:.3f}")
- print(f" hint.source: {hint.source}")
-
- # Print confidence progression
- print("\n--- Confidence Progression ---")
- for i, conf in enumerate(confidence_history):
- print(f" Stage {i + 1}: confidence = {conf:.3f}")
-
- # Verify confidence increases with sample size
- # Confidence should generally increase (may plateau at high values)
- assert confidence_history[-1] >= confidence_history[0], (
- f"Confidence should increase: start={confidence_history[0]:.3f}, "
- f"end={confidence_history[-1]:.3f}"
- )
-
- # With 20 samples, should have meaningful confidence
- assert confidence_history[-1] >= 0.1, (
- f"With 20 samples, confidence should be >= 0.1, got {confidence_history[-1]:.3f}"
- )
-
- print("\n[PASS] Confidence increases with sample size")
-
-
class TestTOINWithSmartCrusher:
"""Test 5: Verify TOIN integration with SmartCrusher."""