Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
test(toin): delete the permanently skipped PR-B5 get_recommendation tests (#4090)
Tejas Chopra85c35799ea1e
diff --git a/tests/test_critical_gaps.py b/tests/test_critical_gaps.pyindex 0ec6fb6cc..9153a99f7 100644--- a/tests/test_critical_gaps.py+++ b/tests/test_critical_gaps.py@@ -845,12 +845,6 @@ class TestTOINHighPriorityFixes:# (auto-save happens inside record_compression)assert not toin._dirty- @pytest.mark.skip(- reason="PR-B5: get_recommendation retired; preserve_fields lives on the aggregated ToolPattern instead"- )- def test_toin_preserves_fields_returns_list(self):- """Retired in PR-B5 along with the request-time hint API."""-class TestCompressionStoreHighPriorityFixes:"""Additional HIGH priority tests for CompressionStore."""diff --git a/tests/test_toin.py b/tests/test_toin.pyindex 40205e339..acacc7e20 100644--- a/tests/test_toin.py+++ b/tests/test_toin.py@@ -1,9 +1,8 @@"""Tests for Tool Output Intelligence Network (TOIN).-PR-B5 retired the request-time hint API. Tests that exercised the old-`get_recommendation()` / `CompressionHint` shape are skipped at module-level — the new observation-only contract is covered by-`tests/test_toin_observation_only.py` and `tests/test_toin_publish.py`.+PR-B5 retired the request-time hint API. The observation-only contract+is covered by `tests/test_toin_observation_only.py` and+`tests/test_toin_publish.py`."""import os@@ -331,47 +330,6 @@ class TestToolIntelligenceNetwork:# Field should be in commonly_retrieved_fields after 3+ retrievalsassert len(pattern.commonly_retrieved_fields) > 0- # PR-B5: the following tests exercised the request-time hint API- # that's now retired. They're skipped wholesale; the new contract- # ("get_recommendation always returns None and emits a deprecation- # warning") is covered by tests/test_toin_observation_only.py.-- @pytest.mark.skip(- reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"- )- def test_get_recommendation_no_data(self):- pass-- @pytest.mark.skip(- reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"- )- def test_get_recommendation_insufficient_samples(self):- pass-- @pytest.mark.skip(- reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"- )- def test_get_recommendation_aggressive_compression(self):- pass-- @pytest.mark.skip(- reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"- )- def test_get_recommendation_conservative_compression(self):- pass-- @pytest.mark.skip(- reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"- )- def test_get_recommendation_skip_compression(self):- pass-- @pytest.mark.skip(- reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"- )- def test_get_recommendation_disabled(self):- pass-def test_get_stats(self):"""get_stats returns overall statistics."""toin = ToolIntelligenceNetwork()diff --git a/tests/test_toin_feedback.py b/tests/test_toin_feedback.pyindex 46ad62a92..7ff35d5ab 100644--- a/tests/test_toin_feedback.py+++ b/tests/test_toin_feedback.py@@ -10,7 +10,6 @@ from headroom.cache.compression_store import get_compression_store, reset_comprefrom headroom.telemetry import (TOINConfig,ToolIntelligenceNetwork,- ToolPattern,ToolSignature,get_toin,reset_toin,@@ -81,94 +80,6 @@ def test_kompress_ccr_retrieval_updates_toin():assert stats["total_retrievals"] == 1[email protected](reason="PR-B5: observations counter and request-time hint API retired")-class TestGetRecommendationObservations:- """Bug 1: get_recommendation() should increment observations counter."""-- def test_increments_observations_when_pattern_exists(self):- """get_recommendation() should increment observations when pattern exists."""- config = _make_config(min_samples=5)- toin = ToolIntelligenceNetwork(config=config)- sig = _make_signature("obs_test_hash")-- # Record enough compressions to create a pattern with sufficient samples- for _ in range(15):- toin.record_compression(- tool_signature=sig,- original_count=100,- compressed_count=20,- original_tokens=5000,- compressed_tokens=1000,- strategy="top_n",- )-- # Get recommendation- toin.get_recommendation(sig)-- # Check observations incremented- pattern = toin._patterns[("global", "unknown", "unknown", sig.structure_hash)]- assert pattern.observations == 1-- # Call again- toin.get_recommendation(sig)- assert pattern.observations == 2-- def test_increments_observations_even_below_min_samples(self):- """observations increments even when sample_size < min_samples."""- config = _make_config(min_samples=100)- toin = ToolIntelligenceNetwork(config=config)- sig = _make_signature("low_sample_hash")-- # Record just a few compressions (below min_samples)- for _ in range(3):- toin.record_compression(- tool_signature=sig,- original_count=50,- compressed_count=10,- original_tokens=2000,- compressed_tokens=500,- strategy="top_n",- )-- result = toin.get_recommendation(sig)- assert result.source == "local" # Not enough samples-- pattern = toin._patterns[("global", "unknown", "unknown", sig.structure_hash)]- assert pattern.observations == 1-- def test_no_increment_for_unknown_pattern(self):- """get_recommendation() should NOT increment for unknown patterns."""- config = _make_config()- toin = ToolIntelligenceNetwork(config=config)- sig = _make_signature("nonexistent_hash")-- result = toin.get_recommendation(sig)- assert result.source == "default"- assert result.reason == "No pattern data for this tool type"-- # No pattern exists, nothing to increment- assert "nonexistent_hash" not in toin._patterns-- def test_observations_survives_serialization(self):- """observations field should serialize and deserialize correctly."""- pattern = ToolPattern(- tool_signature_hash="serial_test",- total_compressions=10,- observations=42,- )-- d = pattern.to_dict()- assert d["observations"] == 42-- restored = ToolPattern.from_dict(d)- assert restored.observations == 42-- def test_observations_defaults_to_zero(self):- """observations defaults to 0 for new patterns."""- pattern = ToolPattern(tool_signature_hash="new_pattern")- assert pattern.observations == 0--class TestRecordRetrievalPopulatesFields:"""Bug 1 related: record_retrieval with query_fields populates field data."""diff --git a/tests/test_toin_fixes.py b/tests/test_toin_fixes.pyindex cb32332b8..b463e29a0 100644--- a/tests/test_toin_fixes.py+++ b/tests/test_toin_fixes.py@@ -68,99 +68,6 @@ def fresh_store():reset_compression_store()- reason="PR-B5: strategy-recommendation API retired (get_recommendation returns None)"-)-class TestStrategySuccessRates:- """Test that strategy_success_rates are used in recommendations."""-- def test_recommends_strategy_with_high_success_rate(self, fresh_toin):- """Strategy with success rate >= 0.5 should be recommended."""- items = [{"id": i, "score": 100 - i} for i in range(20)]- signature = ToolSignature.from_items(items)-- # Record compressions to build pattern- for _ in range(10):- fresh_toin.record_compression(- tool_signature=signature,- original_count=20,- compressed_count=10,- original_tokens=2000,- compressed_tokens=1000,- strategy="smart_sample",- )-- # Set high success rate- pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]- pattern.strategy_success_rates["smart_sample"] = 0.8- pattern.optimal_strategy = "smart_sample"-- # Get recommendation- hint = fresh_toin.get_recommendation(signature, "test query")-- assert hint.recommended_strategy == "smart_sample"-- def test_rejects_strategy_with_low_success_rate(self, fresh_toin):- """Strategy with success rate < 0.5 should NOT be recommended."""- items = [{"id": i, "score": 100 - i} for i in range(20)]- signature = ToolSignature.from_items(items)-- # Record compressions- for _ in range(10):- fresh_toin.record_compression(- tool_signature=signature,- original_count=20,- compressed_count=10,- original_tokens=2000,- compressed_tokens=1000,- strategy="bad_strategy",- )-- # Set low success rate- pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]- pattern.strategy_success_rates["bad_strategy"] = 0.2- pattern.optimal_strategy = "bad_strategy"-- # Get recommendation- hint = fresh_toin.get_recommendation(signature, "test query")-- # Should not recommend the bad strategy- assert hint.recommended_strategy != "bad_strategy"- # Confidence should be reduced- assert "low success" in hint.reason.lower()-- def test_finds_best_strategy_when_optimal_is_bad(self, fresh_toin):- """When optimal_strategy has low success, find a better alternative."""- items = [{"id": i, "score": 100 - i} for i in range(20)]- signature = ToolSignature.from_items(items)-- # Record compressions- for _ in range(10):- fresh_toin.record_compression(- tool_signature=signature,- original_count=20,- compressed_count=10,- original_tokens=2000,- compressed_tokens=1000,- strategy="smart_sample",- )-- # Set up multiple strategies with different success rates- pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]- pattern.strategy_success_rates = {- "bad_strategy": 0.2,- "good_strategy": 0.9,- }- pattern.optimal_strategy = "bad_strategy"-- # Get recommendation- hint = fresh_toin.get_recommendation(signature, "test query")-- # Should recommend the better strategy- assert hint.recommended_strategy == "good_strategy"- assert "using good_strategy instead" in hint.reason--class TestPreserveFieldsMerging:"""Test preserve_fields merging in federated learning."""@@ -352,166 +259,6 @@ class TestUserCountTracking:assert pattern.user_count >= 3- reason="PR-B5: get_recommendation retired; field-weighting now consumed only by toin publish"-)-class TestFieldRetrievalFrequencyWeighting:- """Test field_retrieval_frequency weighting in preserve_fields."""-- def test_query_fields_prioritized_in_preserve_fields(self, fresh_toin):- """Fields mentioned in query should be prioritized."""- items = [{"id": i, "status": "ok", "category": f"cat_{i}"} for i in range(20)]- signature = ToolSignature.from_items(items)-- # Build pattern with field retrieval data- for _ in range(10):- fresh_toin.record_compression(- tool_signature=signature,- original_count=20,- compressed_count=10,- original_tokens=2000,- compressed_tokens=1000,- strategy="smart_sample",- )-- # Record retrievals for "status" field- status_hash = fresh_toin._hash_field_name("status")- pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]- pattern.field_retrieval_frequency = {- status_hash: 50,- fresh_toin._hash_field_name("category"): 10,- }- pattern.preserve_fields = [status_hash]-- # Get recommendation with query mentioning "status"- hint = fresh_toin.get_recommendation(signature, "status:error")-- # status hash should be in preserve_fields- assert status_hash in hint.preserve_fields-- def test_preserve_fields_sorted_by_frequency(self, fresh_toin):- """preserve_fields should be sorted by retrieval frequency."""- items = [{"id": i} for i in range(20)]- signature = ToolSignature.from_items(items)-- # Build pattern- for _ in range(10):- fresh_toin.record_compression(- tool_signature=signature,- original_count=20,- compressed_count=10,- original_tokens=2000,- compressed_tokens=1000,- strategy="smart_sample",- )-- pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]- field_a = fresh_toin._hash_field_name("field_a")- field_b = fresh_toin._hash_field_name("field_b")- field_c = fresh_toin._hash_field_name("field_c")-- pattern.field_retrieval_frequency = {- field_a: 10,- field_b: 50, # Most frequent- field_c: 30,- }- pattern.preserve_fields = [field_a, field_b, field_c]-- # Get recommendation (no query context)- hint = fresh_toin.get_recommendation(signature, "")-- # Should be sorted by frequency- if len(hint.preserve_fields) >= 3:- # field_b should come before field_c which should come before field_a- b_idx = hint.preserve_fields.index(field_b) if field_b in hint.preserve_fields else -1- c_idx = hint.preserve_fields.index(field_c) if field_c in hint.preserve_fields else -1- hint.preserve_fields.index(field_a) if field_a in hint.preserve_fields else -1-- if b_idx >= 0 and c_idx >= 0:- assert b_idx < c_idx, "Higher frequency field should come first"--[email protected](reason="PR-B5: get_recommendation retired (returns None / DeprecationWarning)")-class TestQueryContextUsage:- """Test query_context usage in recommendations."""-- def test_exhaustive_query_keywords_detected(self, fresh_toin):- """Exhaustive query keywords should trigger conservative compression."""- items = [{"id": i, "score": 100 - i} for i in range(50)]- signature = ToolSignature.from_items(items)-- # Build pattern with aggressive compression normally- for _ in range(10):- fresh_toin.record_compression(- tool_signature=signature,- original_count=50,- compressed_count=10,- original_tokens=5000,- compressed_tokens=1000,- strategy="smart_sample",- )-- # Low retrieval rate = aggressive compression- pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]- pattern.total_retrievals = 0-- # Query with exhaustive keyword- hint = fresh_toin.get_recommendation(signature, "list all items in category")-- # Should be more conservative- assert hint.max_items >= 40- assert "exhaustive query" in hint.reason.lower()- assert hint.compression_level == "conservative"-- def test_every_keyword_triggers_conservative(self, fresh_toin):- """'every' keyword should trigger conservative compression."""- items = [{"id": i} for i in range(50)]- signature = ToolSignature.from_items(items)-- for _ in range(10):- fresh_toin.record_compression(- tool_signature=signature,- original_count=50,- compressed_count=10,- original_tokens=5000,- compressed_tokens=1000,- strategy="smart_sample",- )-- pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]- pattern.total_retrievals = 0-- hint = fresh_toin.get_recommendation(signature, "find every user")-- assert "exhaustive query" in hint.reason.lower()-- def test_partial_pattern_matching(self, fresh_toin):- """Partial pattern matching should boost max_items."""- items = [{"id": i, "status": "ok"} for i in range(50)]- signature = ToolSignature.from_items(items)-- for _ in range(10):- fresh_toin.record_compression(- tool_signature=signature,- original_count=50,- compressed_count=10,- original_tokens=5000,- compressed_tokens=1000,- strategy="smart_sample",- )-- pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]- pattern.total_retrievals = 0- # Add a problematic query pattern- pattern.common_query_patterns = ["status:*"]-- # Query that uses the same field- hint = fresh_toin.get_recommendation(signature, "status:error")-- # Should match the pattern- assert hint.max_items >= 25 or "retrieval pattern" in hint.reason--class TestFeedbackStrategyTracking:"""Test strategy tracking in compression feedback."""diff --git a/tests/test_toin_full_integration.py b/tests/test_toin_full_integration.pyindex a74086a54..4d56f2d92 100644--- a/tests/test_toin_full_integration.py+++ b/tests/test_toin_full_integration.py@@ -247,208 +247,6 @@ class TestTOINPersistenceAcrossInstances:print("\n[PASS] TOIN persistence works correctly")- reason="PR-B5: get_recommendation retired; feedback-loop covered by test_toin_observation_only.py"-)-class TestTOINFullFeedbackLoop:- """Test 3: Verify TOIN feedback loop with recommendations."""-- def test_toin_full_feedback_loop(self, sample_tool_signature):- """Verify TOIN learns from high retrieval rate and recommends skip."""- print("\n" + "=" * 60)- print("TEST: test_toin_full_feedback_loop")- print("=" * 60)-- with tempfile.TemporaryDirectory() as tmpdir:- storage_path = str(Path(tmpdir) / "toin_feedback_test.json")- config = TOINConfig(- storage_path=storage_path,- min_samples_for_recommendation=5, # Lower threshold for test- high_retrieval_threshold=0.5, # 50% retrieval = high- )- toin = ToolIntelligenceNetwork(config)-- print("\n--- Phase 1: Record compressions ---")- # Record 5 compressions with same tool signature- for i in range(5):- toin.record_compression(- tool_signature=sample_tool_signature,- original_count=100,- compressed_count=15,- original_tokens=5000,- compressed_tokens=750,- strategy="smart_sample",- )- print(f" Recorded compression {i + 1}")-- print("\n--- Phase 2: Record retrievals (simulating high retrieval rate) ---")- # Record 3 full retrievals (60% retrieval rate = high)- for i in range(3):- toin.record_retrieval(- tool_signature_hash=sample_tool_signature.structure_hash,- retrieval_type="full", # Full retrieval = compression too aggressive- strategy="smart_sample",- )- print(f" Recorded full retrieval {i + 1}")-- # Get pattern stats- pattern = toin.get_pattern(sample_tool_signature.structure_hash)- print("\n--- Pattern Stats ---")- print(f" total_compressions: {pattern.total_compressions}")- print(f" total_retrievals: {pattern.total_retrievals}")- print(f" retrieval_rate: {pattern.retrieval_rate:.1%}")- print(f" full_retrieval_rate: {pattern.full_retrieval_rate:.1%}")- print(f" skip_compression_recommended: {pattern.skip_compression_recommended}")-- # Get recommendation- print("\n--- Getting Recommendation ---")- hint = toin.get_recommendation(sample_tool_signature)- print(f" source: {hint.source}")- print(f" skip_compression: {hint.skip_compression}")- print(f" compression_level: {hint.compression_level}")- print(f" max_items: {hint.max_items}")- print(f" confidence: {hint.confidence:.3f}")- print(f" reason: {hint.reason}")- print(f" based_on_samples: {hint.based_on_samples}")-- # Verify high retrieval rate triggers skip recommendation- # With 60% retrieval rate (3/5) and full_retrieval_rate of 100% (3/3),- # TOIN should recommend skipping compression- retrieval_rate = pattern.retrieval_rate- assert retrieval_rate >= 0.5, (- f"Expected retrieval rate >= 50%, got {retrieval_rate:.1%}"- )-- # With high retrieval rate and high full retrieval rate, should skip- if pattern.full_retrieval_rate > 0.8:- assert hint.skip_compression or hint.compression_level in (- "none",- "conservative",- ), (- f"High full retrieval rate should trigger skip or conservative, "- f"got compression_level={hint.compression_level}"- )- print("\n[PASS] High retrieval rate correctly influences recommendation")- else:- print("\n[INFO] Full retrieval rate not high enough for skip recommendation")- print(f" full_retrieval_rate: {pattern.full_retrieval_rate:.1%}")-- print("\n[PASS] TOIN feedback loop works correctly")--- reason="PR-B5: get_recommendation retired; confidence-progression validated via record + get_pattern instead"-)-class TestTOINProgressiveConfidence:- """Test 4: Verify TOIN confidence increases with sample size."""-- def test_toin_progressive_confidence(self, sample_tool_signature):- """Verify confidence increases with more samples."""- print("\n" + "=" * 60)- print("TEST: test_toin_progressive_confidence")- print("=" * 60)-- with tempfile.TemporaryDirectory() as tmpdir:- storage_path = str(Path(tmpdir) / "toin_confidence_test.json")- config = TOINConfig(- storage_path=storage_path,- min_samples_for_recommendation=3,- )- toin = ToolIntelligenceNetwork(config)-- confidence_history = []-- # Batch 1: Record 1 compression- print("\n--- Batch 1: 1 compression ---")- toin.record_compression(- tool_signature=sample_tool_signature,- original_count=100,- compressed_count=15,- original_tokens=5000,- compressed_tokens=750,- strategy="smart_sample",- )- pattern = toin.get_pattern(sample_tool_signature.structure_hash)- hint = toin.get_recommendation(sample_tool_signature)- confidence_history.append(pattern.confidence)- print(f" sample_size: {pattern.sample_size}")- print(f" confidence: {pattern.confidence:.3f}")- print(f" hint.source: {hint.source}")-- # Batch 2: Record 2 more compressions- print("\n--- Batch 2: +2 compressions (total: 3) ---")- for _ in range(2):- toin.record_compression(- tool_signature=sample_tool_signature,- original_count=100,- compressed_count=15,- original_tokens=5000,- compressed_tokens=750,- strategy="smart_sample",- )- pattern = toin.get_pattern(sample_tool_signature.structure_hash)- hint = toin.get_recommendation(sample_tool_signature)- confidence_history.append(pattern.confidence)- print(f" sample_size: {pattern.sample_size}")- print(f" confidence: {pattern.confidence:.3f}")- print(f" hint.source: {hint.source}")-- # Batch 3: Record 2 more compressions- print("\n--- Batch 3: +2 compressions (total: 5) ---")- for _ in range(2):- toin.record_compression(- tool_signature=sample_tool_signature,- original_count=100,- compressed_count=15,- original_tokens=5000,- compressed_tokens=750,- strategy="smart_sample",- )- pattern = toin.get_pattern(sample_tool_signature.structure_hash)- hint = toin.get_recommendation(sample_tool_signature)- confidence_history.append(pattern.confidence)- print(f" sample_size: {pattern.sample_size}")- print(f" confidence: {pattern.confidence:.3f}")- print(f" hint.source: {hint.source}")-- # Batch 4: Add many more to boost confidence- print("\n--- Batch 4: +15 compressions (total: 20) ---")- for _ in range(15):- toin.record_compression(- tool_signature=sample_tool_signature,- original_count=100,- compressed_count=15,- original_tokens=5000,- compressed_tokens=750,- strategy="smart_sample",- )- pattern = toin.get_pattern(sample_tool_signature.structure_hash)- hint = toin.get_recommendation(sample_tool_signature)- confidence_history.append(pattern.confidence)- print(f" sample_size: {pattern.sample_size}")- print(f" confidence: {pattern.confidence:.3f}")- print(f" hint.source: {hint.source}")-- # Print confidence progression- print("\n--- Confidence Progression ---")- for i, conf in enumerate(confidence_history):- print(f" Stage {i + 1}: confidence = {conf:.3f}")-- # Verify confidence increases with sample size- # Confidence should generally increase (may plateau at high values)- assert confidence_history[-1] >= confidence_history[0], (- f"Confidence should increase: start={confidence_history[0]:.3f}, "- f"end={confidence_history[-1]:.3f}"- )-- # With 20 samples, should have meaningful confidence- assert confidence_history[-1] >= 0.1, (- f"With 20 samples, confidence should be >= 0.1, got {confidence_history[-1]:.3f}"- )-- print("\n[PASS] Confidence increases with sample size")--class TestTOINWithSmartCrusher:"""Test 5: Verify TOIN integration with SmartCrusher."""