The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
"""
Unit tests for Compression Interception Handler.
"""
from unittest.mock import MagicMock
import pytest
from litellm.integrations.compression_interception.handler import (
CompressionInterceptionLogger,
)
from litellm.types.utils import CallTypes
def test_initialize_from_proxy_config():
"""Test initialization from proxy config with litellm_settings."""
litellm_settings = {
"compression_interception_params": {
"enabled": True,
"compression_trigger": 1234,
"compression_target": 789,
}
}
logger = CompressionInterceptionLogger.initialize_from_proxy_config(
litellm_settings=litellm_settings,
callback_specific_params={},
)
assert logger.enabled is True
assert logger.compression_trigger == 1234
assert logger.compression_target == 789
def test_initialize_from_proxy_config_ignores_non_dict_callback_specific_params():
"""Regression (#29590): a non-dict value under
callback_settings.compression_interception must not crash initialization.
Forwarding callback_settings as callback_specific_params activates this
branch; without the isinstance(dict) guard a non-dict value reached
from_config_yaml(...).get(...) and raised AttributeError at proxy startup.
The value is ignored and the logger falls back to defaults.
"""
logger = CompressionInterceptionLogger.initialize_from_proxy_config(
litellm_settings={},
callback_specific_params={"compression_interception": True},
)
assert logger.enabled is True
assert logger.compression_trigger == 200_000
def test_initialize_from_proxy_config_honors_dict_callback_specific_params():
"""A valid dict under callback_settings.compression_interception is applied."""
logger = CompressionInterceptionLogger.initialize_from_proxy_config(
litellm_settings={},
callback_specific_params={
"compression_interception": {
"enabled": False,
"compression_trigger": 12345,
}
},
)
assert logger.enabled is False
assert logger.compression_trigger == 12345
@pytest.mark.asyncio
async def test_pre_call_hook_compresses_messages_and_injects_tool(monkeypatch):
"""Test pre-call hook compresses and stores per-call cache."""
logger = CompressionInterceptionLogger()
compressed_result = {
"messages": [{"role": "user", "content": "stubbed"}],
"original_tokens": 12000,
"compressed_tokens": 5000,
"compression_ratio": 0.58,
"cache": {"auth.py": "full file content"},
"tools": [
{
"type": "function",
"function": {
"name": "litellm_content_retrieve",
"parameters": {
"type": "object",
"properties": {"key": {"type": "string"}},
},
},
}
],
}
def _fake_compress(**kwargs):
return compressed_result
# The handler does ``from litellm.compression import compress`` at module
# scope, so we must patch the binding on the handler module — patching
# ``litellm.compress`` has no effect on the already-bound reference.
monkeypatch.setattr(
"litellm.integrations.compression_interception.handler.compress",
_fake_compress,
)
kwargs = {
"model": "bedrock/us.anthropic.claude-sonnet-4-5",
"messages": [{"role": "user", "content": "very large context"}],
"tools": [
{
"type": "function",
"function": {"name": "existing_tool", "parameters": {"type": "object"}},
}
],
}
result = await logger.async_pre_call_deployment_hook(
kwargs=kwargs, call_type=CallTypes.anthropic_messages
)
assert result is not None
assert result["messages"] == compressed_result["messages"]
tool_names = [t.get("function", {}).get("name") for t in result["tools"]]
assert "existing_tool" in tool_names
assert "litellm_content_retrieve" in tool_names
assert result["litellm_call_id"] in logger._compression_cache_by_call_id
@pytest.mark.asyncio
async def test_pre_call_hook_below_trigger_does_not_inject_empty_tools(monkeypatch):
"""
When compression is a no-op (below trigger / invalid tool sequence), the
hook must NOT replace ``messages`` or inject an empty ``tools: []`` onto
a request that originally had no tools — Anthropic Messages rejects
``tools: []``.
"""
logger = CompressionInterceptionLogger()
original_messages = [{"role": "user", "content": "short prompt"}]
def _fake_compress_noop(**kwargs):
return {
"messages": original_messages,
"original_tokens": 42,
"compressed_tokens": 42,
"compression_ratio": 0.0,
"cache": {},
"tools": [],
"compression_skipped_reason": "below_trigger",
}
monkeypatch.setattr(
"litellm.integrations.compression_interception.handler.compress",
_fake_compress_noop,
)
kwargs = {
"model": "bedrock/us.anthropic.claude-sonnet-4-5",
"messages": original_messages,
}
result = await logger.async_pre_call_deployment_hook(
kwargs=kwargs, call_type=CallTypes.anthropic_messages
)
assert result is not None
# Original request had no ``tools`` — skipped compression must leave it that way.
assert "tools" not in result
# Cache must not be populated for a no-op.
assert result.get("litellm_call_id") not in logger._compression_cache_by_call_id
@pytest.mark.asyncio
async def test_should_run_agentic_loop_detects_retrieval_tool_use():
"""Test should-run hook returns tool calls for retrieval tool_use blocks."""
logger = CompressionInterceptionLogger()
response = {
"content": [
{
"type": "tool_use",
"id": "toolu_123",
"name": "litellm_content_retrieve",
"input": {"key": "auth.py"},
}
]
}
should_run, tools_dict = await logger.async_should_run_agentic_loop(
response=response,
model="bedrock/claude",
messages=[],
tools=[
{
"type": "function",
"function": {
"name": "litellm_content_retrieve",
"parameters": {"type": "object"},
},
}
],
stream=False,
custom_llm_provider="bedrock",
kwargs={},
)
assert should_run is True
assert len(tools_dict["tool_calls"]) == 1
assert tools_dict["tool_calls"][0]["input"]["key"] == "auth.py"
@pytest.mark.asyncio
async def test_build_agentic_loop_plan_returns_request_patch():
"""Callback should return typed patch with tool_result content."""
logger = CompressionInterceptionLogger()
call_id = "call_123"
logger._compression_cache_by_call_id[call_id] = (
{"auth.py": "full auth file"},
9999999999.0,
)
logging_obj = MagicMock()
logging_obj.litellm_call_id = call_id
logging_obj.model_call_details = {
"agentic_loop_params": {"model": "bedrock/invoke/claude-3-5-sonnet"}
}
plan = await logger.async_build_agentic_loop_plan(
tools={
"tool_calls": [
{
"id": "toolu_abc",
"type": "tool_use",
"name": "litellm_content_retrieve",
"input": {"key": "auth.py"},
}
]
},
model="claude-3-5-sonnet",
messages=[{"role": "user", "content": "read auth.py"}],
response=None,
anthropic_messages_provider_config=None,
anthropic_messages_optional_request_params={
"max_tokens": 1024,
"tools": [{"name": "litellm_content_retrieve"}],
},
logging_obj=logging_obj,
stream=False,
kwargs={
"temperature": 0.1,
"_compression_interception_internal": True,
"litellm_logging_obj": object(),
},
)
assert plan.run_agentic_loop is True
assert plan.request_patch is not None
assert plan.request_patch.model == "bedrock/invoke/claude-3-5-sonnet"
assert plan.request_patch.max_tokens == 1024
assert plan.request_patch.messages is not None
assert len(plan.request_patch.messages) == 3
tool_result_content = plan.request_patch.messages[-1]["content"][0]["content"]
assert tool_result_content == "full auth file"
assert "_compression_interception_internal" not in plan.request_patch.kwargs
assert "litellm_logging_obj" not in plan.request_patch.kwargs
assert plan.request_patch.kwargs["temperature"] == 0.1
assert "max_tokens" not in plan.request_patch.optional_params
@pytest.mark.asyncio
async def test_should_run_agentic_loop_with_custom_type_tools():
"""Test that async_should_run_agentic_loop returns True when tools contain
litellm_content_retrieve as a custom-typed tool (e.g. Claude Code tool list)
and the model response includes a matching tool_use block."""
logger = CompressionInterceptionLogger()
# Exact tools payload produced by Claude Code – litellm_content_retrieve is
# the final entry and uses type="custom" (not type="function").
tools = [
{
"name": "Agent",
"description": "Launch a new agent to handle complex, multi-step tasks.",
"input_schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"description": {"type": "string"},
"prompt": {"type": "string"},
},
"required": ["description", "prompt"],
"additionalProperties": False,
},
},
{
"name": "AskUserQuestion",
"description": "Use this tool when you need to ask the user questions.",
"input_schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"questions": {"type": "array", "items": {"type": "object"}},
},
"required": ["questions"],
"additionalProperties": False,
},
},
{
"name": "Bash",
"description": "Executes a given bash command and returns its output.",
"input_schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {"command": {"type": "string"}},
"required": ["command"],
"additionalProperties": False,
},
},
{
"name": "litellm_content_retrieve",
"description": "Retrieve the full content of a file or message that was compressed to save tokens.",
"input_schema": {
"type": "object",
"properties": {
"key": {
"type": "string",
"description": "The identifier of the content to retrieve",
"enum": [
"message_0",
"HA_UPTIME_ROUTER_SPEC.md",
"message_159",
"message_160",
],
}
},
"required": ["key"],
},
"type": "custom",
},
]
response = {
"content": [
{
"type": "tool_use",
"id": "toolu_abc",
"name": "litellm_content_retrieve",
"input": {"key": "message_0"},
}
]
}
should_run, tools_dict = await logger.async_should_run_agentic_loop(
response=response,
model="claude-3-5-sonnet",
messages=[],
tools=tools,
stream=False,
custom_llm_provider="anthropic",
kwargs={},
)
assert should_run is True
assert tools_dict["tool_type"] == "compression_retrieval"
assert len(tools_dict["tool_calls"]) == 1
assert tools_dict["tool_calls"][0]["input"]["key"] == "message_0"
@pytest.mark.asyncio
async def test_build_agentic_loop_plan_missing_key_fallback():
"""Missing cache keys should produce deterministic fallback content."""
logger = CompressionInterceptionLogger()
logging_obj = MagicMock()
logging_obj.litellm_call_id = "missing_call"
logging_obj.model_call_details = {"agentic_loop_params": {}}
plan = await logger.async_build_agentic_loop_plan(
tools={
"tool_calls": [
{
"id": "toolu_missing",
"type": "tool_use",
"name": "litellm_content_retrieve",
"input": {"key": "not_found.py"},
}
]
},
model="claude-3-5-sonnet",
messages=[{"role": "user", "content": "read file"}],
response=None,
anthropic_messages_provider_config=None,
anthropic_messages_optional_request_params={},
logging_obj=logging_obj,
stream=False,
kwargs={},
)
assert plan.request_patch is not None
assert (
plan.request_patch.messages[-1]["content"][0]["content"]
== "[compressed content key 'not_found.py' not found]"
)
def _stub_compress_result(original_tokens, compressed_tokens, cache):
return {
"messages": [{"role": "user", "content": "stubbed"}],
"original_tokens": original_tokens,
"compressed_tokens": compressed_tokens,
"compression_ratio": 0.5,
"cache": cache,
"tools": [],
}
@pytest.mark.asyncio
async def test_pre_call_hook_records_compression_savings_in_litellm_metadata(monkeypatch):
"""
When compression fires, the hook must record tokens_before/after/saved into
the request's litellm_metadata IN PLACE (the proxy and logging object hold
references to the same dict), so the savings land in the SpendLog row's
metadata JSON under ``compression_savings``.
"""
logger = CompressionInterceptionLogger()
monkeypatch.setattr(
"litellm.integrations.compression_interception.handler.compress",
lambda **kwargs: _stub_compress_result(12000, 5000, {"auth.py": "content"}),
)
litellm_metadata = {"user_api_key": "hashed-key", "user_api_key_user_id": "u1"}
kwargs = {
"model": "claude-sonnet-5",
"messages": [{"role": "user", "content": "very large context"}],
"litellm_metadata": litellm_metadata,
}
result = await logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages)
assert result is not None
assert result["litellm_metadata"] is litellm_metadata
assert litellm_metadata["compression_savings"] == {
"tokens_before": 12000,
"tokens_after": 5000,
"tokens_saved": 7000,
"source": "compression_interception",
}
assert litellm_metadata["user_api_key"] == "hashed-key"
@pytest.mark.asyncio
async def test_pre_call_hook_creates_litellm_metadata_when_absent(monkeypatch):
"""SDK-direct calls have no litellm_metadata dict yet; the hook creates it."""
logger = CompressionInterceptionLogger()
monkeypatch.setattr(
"litellm.integrations.compression_interception.handler.compress",
lambda **kwargs: _stub_compress_result(300, 100, {"k": "v"}),
)
kwargs = {
"model": "claude-sonnet-5",
"messages": [{"role": "user", "content": "ctx"}],
}
result = await logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages)
assert result is not None
assert result["litellm_metadata"]["compression_savings"]["tokens_saved"] == 200
@pytest.mark.asyncio
@pytest.mark.parametrize(
"original_tokens,compressed_tokens",
[
(None, 5000),
(12000, None),
("12000", 5000),
(12000, "5000"),
(True, 5000),
(5000, 12000),
(12000, -1),
],
)
async def test_pre_call_hook_invalid_token_counts_fail_open(monkeypatch, original_tokens, compressed_tokens):
"""Invalid token counts must never crash the request; savings simply are not recorded."""
logger = CompressionInterceptionLogger()
monkeypatch.setattr(
"litellm.integrations.compression_interception.handler.compress",
lambda **kwargs: _stub_compress_result(original_tokens, compressed_tokens, {"k": "v"}),
)
litellm_metadata = {"user_api_key": "hashed-key"}
kwargs = {
"model": "claude-sonnet-5",
"messages": [{"role": "user", "content": "ctx"}],
"litellm_metadata": litellm_metadata,
}
result = await logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages)
assert result is not None
assert "compression_savings" not in litellm_metadata
@pytest.mark.asyncio
async def test_pre_call_hook_no_compression_records_no_savings(monkeypatch):
"""When compression is a no-op (empty cache) nothing is recorded."""
logger = CompressionInterceptionLogger()
monkeypatch.setattr(
"litellm.integrations.compression_interception.handler.compress",
lambda **kwargs: {
"messages": [],
"original_tokens": 100,
"compressed_tokens": 100,
"cache": {},
"tools": [],
"compression_skipped_reason": "below_trigger",
},
)
litellm_metadata = {"user_api_key": "hashed-key"}
kwargs = {
"model": "claude-sonnet-5",
"messages": [{"role": "user", "content": "small"}],
"litellm_metadata": litellm_metadata,
}
await logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages)
assert "compression_savings" not in litellm_metadata
@pytest.mark.asyncio
async def test_pre_call_hook_counts_tokens_off_the_event_loop():
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
warm_tokenizer,
)
model = "anthropic/claude-fable-5"
warm_tokenizer(model)
logger = CompressionInterceptionLogger(compression_trigger=10_000_000)
messages = [{"role": "user", "content": text * 100}]
kwargs = {"model": model, "messages": messages}
result, took, lags = await timed_with_loop_lags(
lambda: logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages)
)
assert result is not None
assert result["messages"] is messages
assert "tools" not in result
assert_loop_stayed_free(took, lags)