Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

67.5 KB
Raw Blame
"""
Test priority-based rate limiting for dynamic_rate_limiter_v3.

Core tests to validate that priority weights are respected (0.9/0.1) instead of equal splitting (0.5/0.5).
"""

import asyncio
import os
import time
from datetime import datetime, timedelta
from unittest.mock import AsyncMock, patch

import pytest


import litellm
from litellm import DualCache, Router
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.hooks.dynamic_rate_limiter_v3 import (
    PROXY_DynamicRateLimitHandlerV3 as DynamicRateLimitHandler,
)


class TimeController:
    def __init__(self):
        self._current = datetime.utcnow()

    def now(self) -> datetime:
        return self._current

    def advance(self, seconds: float) -> None:
        self._current += timedelta(seconds=seconds)


@pytest.fixture
def time_controller(monkeypatch):
    controller = TimeController()
    monkeypatch.setattr(time, "time", lambda: controller.now().timestamp())
    return controller


@pytest.mark.asyncio
async def test_priority_weight_allocation(monkeypatch):
    """
    Test that priority weights are correctly applied instead of equal splitting.

    With priority_reservation = {"high": 0.9, "low": 0.1}:
    - High priority should get 90% of TPM (900 out of 1000)
    - Low priority should get 10% of TPM (100 out of 1000)

    This validates the core fix where before it would split 50/50.
    """
    # Set up environment for premium feature
    monkeypatch.setenv("LITELLM_LICENSE", "test-license-key")

    # Set up priority reservations
    litellm.priority_reservation = {"high": 0.9, "low": 0.1}

    dual_cache = DualCache()
    handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)

    model = "test-model"
    total_tpm = 1000

    llm_router = Router(
        model_list=[
            {
                "model_name": model,
                "litellm_params": {
                    "model": "gpt-3.5-turbo",
                    "api_key": "test-key",
                    "api_base": "test-base",
                    "tpm": total_tpm,
                },
            }
        ]
    )
    handler.update_variables(llm_router=llm_router)

    # Test high priority allocation
    high_priority_user = UserAPIKeyAuth()
    high_priority_user.metadata = {"priority": "high"}

    high_descriptors = handler._create_priority_based_descriptors(
        model=model,
        user_api_key_dict=high_priority_user,
        priority="high",
    )

    assert len(high_descriptors) == 1
    high_descriptor = high_descriptors[0]
    expected_high_tpm = int(total_tpm * 0.9)  # 900
    actual_high_tpm = high_descriptor["rate_limit"]["tokens_per_unit"]

    assert (
        actual_high_tpm == expected_high_tpm
    ), f"High priority should get {expected_high_tpm} TPM (90%), got {actual_high_tpm}"
    assert high_descriptor["value"] == f"{model}:high"

    # Test low priority allocation
    low_priority_user = UserAPIKeyAuth()
    low_priority_user.metadata = {"priority": "low"}

    low_descriptors = handler._create_priority_based_descriptors(
        model=model,
        user_api_key_dict=low_priority_user,
        priority="low",
    )

    assert len(low_descriptors) == 1
    low_descriptor = low_descriptors[0]
    expected_low_tpm = int(total_tpm * 0.1)  # 100
    actual_low_tpm = low_descriptor["rate_limit"]["tokens_per_unit"]

    assert (
        actual_low_tpm == expected_low_tpm
    ), f"Low priority should get {expected_low_tpm} TPM (10%), got {actual_low_tpm}"
    assert low_descriptor["value"] == f"{model}:low"

    # Verify the ratio is 9:1, not 1:1 (equal splitting)
    ratio = actual_high_tpm / actual_low_tpm
    expected_ratio = 9.0
    assert (
        abs(ratio - expected_ratio) < 0.1
    ), f"High:Low ratio should be {expected_ratio}:1, got {ratio}:1"


@pytest.mark.asyncio
async def test_concurrent_priority_requests(monkeypatch):
    """
    Test the core issue: 5 concurrent requests with different priorities should get
    proper allocation based on priority weights, not equal splitting.

    This tests the exact scenario mentioned: priorities 0.9 and 0.1 should be 0.9/0.1, not 0.5/0.5.
    """
    # Set up environment for premium feature
    monkeypatch.setenv("LITELLM_LICENSE", "test-license-key")

    # Set up the exact scenario from the issue
    litellm.priority_reservation = {"high": 0.9, "low": 0.1}

    dual_cache = DualCache()
    handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)

    model = "test-model"
    total_tpm = 1000

    llm_router = Router(
        model_list=[
            {
                "model_name": model,
                "litellm_params": {
                    "model": "gpt-3.5-turbo",
                    "api_key": "test-key",
                    "api_base": "test-base",
                    "tpm": total_tpm,
                },
            }
        ]
    )
    handler.update_variables(llm_router=llm_router)

    # Create 5 concurrent users - 3 high priority, 2 low priority
    high_priority_users = []
    low_priority_users = []

    for i in range(3):  # 3 high priority users
        user = UserAPIKeyAuth()
        user.metadata = {"priority": "high"}
        user.user_id = f"high_user_{i}"
        high_priority_users.append(user)

    for i in range(2):  # 2 low priority users
        user = UserAPIKeyAuth()
        user.metadata = {"priority": "low"}
        user.user_id = f"low_user_{i}"
        low_priority_users.append(user)

    # Test all high priority users get the same allocation (not divided)
    for user in high_priority_users:
        descriptors = handler._create_priority_based_descriptors(
            model=model,
            user_api_key_dict=user,
            priority="high",
        )

        assert len(descriptors) == 1
        descriptor = descriptors[0]
        # Each high priority user should get 900 TPM, not divided by 3
        assert descriptor["rate_limit"]["tokens_per_unit"] == 900, (
            f"High priority user {user.user_id} should get 900 TPM, "
            f"got {descriptor['rate_limit']['tokens_per_unit']}"
        )
        assert descriptor["value"] == f"{model}:high"

    # Test all low priority users get the same allocation (not divided)
    for user in low_priority_users:
        descriptors = handler._create_priority_based_descriptors(
            model=model,
            user_api_key_dict=user,
            priority="low",
        )

        assert len(descriptors) == 1
        descriptor = descriptors[0]
        # Each low priority user should get 100 TPM, not divided by 2
        assert descriptor["rate_limit"]["tokens_per_unit"] == 100, (
            f"Low priority user {user.user_id} should get 100 TPM, "
            f"got {descriptor['rate_limit']['tokens_per_unit']}"
        )
        assert descriptor["value"] == f"{model}:low"


@pytest.mark.asyncio
async def test_100_concurrent_priority_requests(time_controller, monkeypatch):
    """
    Stress test: 100 concurrent requests with mixed priorities over 10 seconds.

    This validates that the priority system works correctly under high load:
    - 70 high priority requests (should get 900 TPM each)
    - 30 low priority requests (should get 100 TPM each)
    - Spread across 10 seconds to simulate real-world load
    """
    # Set up environment for premium feature
    monkeypatch.setenv("LITELLM_LICENSE", "test-license-key")

    # Set up priority reservations
    litellm.priority_reservation = {"high": 0.9, "low": 0.1}

    dual_cache = DualCache()
    handler = DynamicRateLimitHandler(
        internal_usage_cache=dual_cache, time_provider=time_controller.now
    )

    model = "stress-test-model"
    total_tpm = 1000

    llm_router = Router(
        model_list=[
            {
                "model_name": model,
                "litellm_params": {
                    "model": "gpt-3.5-turbo",
                    "api_key": "test-key",
                    "api_base": "test-base",
                    "tpm": total_tpm,
                    "rpm": 500,  # Also test RPM limits
                },
            }
        ]
    )
    handler.update_variables(llm_router=llm_router)

    # Create 100 users: 70 high priority, 30 low priority
    all_users = []

    # 70 high priority users
    for i in range(70):
        user = UserAPIKeyAuth()
        user.metadata = {"priority": "high"}
        user.user_id = f"high_stress_user_{i}"
        all_users.append((user, "high", 900, 450))  # expected TPM, expected RPM

    # 30 low priority users
    for i in range(30):
        user = UserAPIKeyAuth()
        user.metadata = {"priority": "low"}
        user.user_id = f"low_stress_user_{i}"
        all_users.append((user, "low", 100, 50))  # expected TPM, expected RPM

    async def test_user_descriptors(user_data):
        """Test descriptor creation for a single user."""
        user, priority, expected_tpm, expected_rpm = user_data

        descriptors = handler._create_priority_based_descriptors(
            model=model,
            user_api_key_dict=user,
            priority=priority,
        )

        assert (
            len(descriptors) == 1
        ), f"User {user.user_id} should have exactly 1 descriptor"
        descriptor = descriptors[0]

        # Validate TPM allocation
        actual_tpm = descriptor["rate_limit"]["tokens_per_unit"]
        assert (
            actual_tpm == expected_tpm
        ), f"User {user.user_id} ({priority}) should get {expected_tpm} TPM, got {actual_tpm}"

        # Validate RPM allocation
        actual_rpm = descriptor["rate_limit"]["requests_per_unit"]
        assert (
            actual_rpm == expected_rpm
        ), f"User {user.user_id} ({priority}) should get {expected_rpm} RPM, got {actual_rpm}"

        # Validate descriptor key
        assert descriptor["value"] == f"{model}:{priority}"
        assert descriptor["key"] == "priority_model"

        return {
            "user_id": user.user_id,
            "priority": priority,
            "tpm": actual_tpm,
            "rpm": actual_rpm,
            "success": True,
        }

    # Run all 100 requests concurrently to simulate high load
    start_time = time.time()

    # Split into batches to simulate requests over 10 seconds
    batch_size = 10  # 10 requests per batch
    batches = [
        all_users[i : i + batch_size] for i in range(0, len(all_users), batch_size)
    ]

    all_results = []

    for batch_idx, batch in enumerate(batches):
        # Process each batch concurrently
        batch_tasks = [test_user_descriptors(user_data) for user_data in batch]
        batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True)
        all_results.extend(batch_results)

        # Add small delay between batches to spread over ~10 seconds
        if batch_idx < len(batches) - 1:  # Don't sleep after last batch
            await asyncio.sleep(0)
            time_controller.advance(1.0)  # simulate 1s passing between batches

    end_time = time.time()
    total_duration = end_time - start_time

    # Validate that the test ran over approximately 10 seconds
    assert (
        total_duration >= 9.0
    ), f"Test should take ~10 seconds, took {total_duration:.2f}s"
    assert total_duration <= 15.0, f"Test took too long: {total_duration:.2f}s"

    # Validate all requests were successful
    successful_results = [
        r for r in all_results if isinstance(r, dict) and r.get("success")
    ]
    assert (
        len(successful_results) == 100
    ), f"Expected 100 successful results, got {len(successful_results)}"

    # Validate priority distribution
    high_priority_results = [r for r in successful_results if r["priority"] == "high"]
    low_priority_results = [r for r in successful_results if r["priority"] == "low"]

    assert (
        len(high_priority_results) == 70
    ), f"Expected 70 high priority results, got {len(high_priority_results)}"
    assert (
        len(low_priority_results) == 30
    ), f"Expected 30 low priority results, got {len(low_priority_results)}"

    # Validate all high priority users got correct allocation
    for result in high_priority_results:
        assert (
            result["tpm"] == 900
        ), f"High priority user {result['user_id']} got {result['tpm']} TPM, expected 900"
        assert (
            result["rpm"] == 450
        ), f"High priority user {result['user_id']} got {result['rpm']} RPM, expected 450"

    # Validate all low priority users got correct allocation
    for result in low_priority_results:
        assert (
            result["tpm"] == 100
        ), f"Low priority user {result['user_id']} got {result['tpm']} TPM, expected 100"
        assert (
            result["rpm"] == 50
        ), f"Low priority user {result['user_id']} got {result['rpm']} RPM, expected 50"

    print(f"✅ Successfully processed 100 concurrent requests in {total_duration:.2f}s")
    print(f"   - 70 high priority users: 900 TPM, 450 RPM each")
    print(f"   - 30 low priority users: 100 TPM, 50 RPM each")
    print(f"   - Priority ratio maintained: 9:1 (TPM) and 9:1 (RPM)")


@pytest.mark.asyncio
async def test_concurrent_pre_call_hooks_stress(monkeypatch):
    """
    Stress test: 50 concurrent pre-call hooks with saturation-aware priority enforcement.

    Tests priority-based rate limiting in strict mode (>80% saturation).
    Mocks high saturation to force strict mode where priorities are enforced.
    Premium users (80% allocation) should have >90% success rate.
    Standard users (20% allocation) should have ~70% success rate with 30% random limiting.
    """
    # Set up environment for premium feature
    monkeypatch.setenv("LITELLM_LICENSE", "test-license-key")

    litellm.priority_reservation = {"premium": 0.8, "standard": 0.2}

    dual_cache = DualCache()
    handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)

    model = "pre-call-stress-model"
    total_tpm = 2000

    llm_router = Router(
        model_list=[
            {
                "model_name": model,
                "litellm_params": {
                    "model": "gpt-3.5-turbo",
                    "api_key": "test-key",
                    "api_base": "test-base",
                    "tpm": total_tpm,
                },
            }
        ]
    )
    handler.update_variables(llm_router=llm_router)

    # Mock the v3 limiter to simulate different scenarios
    successful_requests = []
    rate_limited_requests = []

    # Mock saturation check to return high saturation (forces strict mode)
    async def mock_get_cache(key, litellm_parent_otel_span=None, local_only=False):
        """Mock cache to simulate high saturation."""
        # Return high usage to trigger strict mode (>80% saturation)
        if ":requests" in key or ":tokens" in key:
            return 1800  # 1800/2000 = 90% saturation
        return None

    async def mock_should_rate_limit(
        descriptors, parent_otel_span=None, read_only=False
    ):
        """Mock rate limiter that handles saturation-aware descriptors."""
        descriptor = descriptors[0]
        descriptor_key = descriptor["key"]
        descriptor_value = descriptor["value"]

        # Handle model-wide tracking (for both generous and strict mode tracking)
        if descriptor_key == "model_saturation_check":
            # Always allow model-wide tracking (doesn't enforce in our mock)
            return {
                "overall_code": "OK",
                "statuses": [
                    {
                        "code": "OK",
                        "descriptor_key": descriptor_value,
                        "rate_limit_type": "tokens_per_unit",
                        "limit_remaining": 10000,
                    }
                ],
            }

        # Handle priority-specific enforcement in strict mode
        elif descriptor_key == "priority_model":
            # Extract priority from value like "pre-call-stress-model:premium"
            priority = descriptor_value.split(":")[-1]

            if priority == "premium":
                # Allow all premium requests
                return {
                    "overall_code": "OK",
                    "statuses": [
                        {
                            "code": "OK",
                            "descriptor_key": descriptor_value,
                            "rate_limit_type": "tokens_per_unit",
                            "limit_remaining": 1000,
                        }
                    ],
                }
            else:
                # Rate limit some standard requests (simulate load)
                import random

                if random.random() < 0.3:  # 30% of standard requests get rate limited
                    return {
                        "overall_code": "OVER_LIMIT",
                        "statuses": [
                            {
                                "code": "OVER_LIMIT",
                                "descriptor_key": descriptor_value,
                                "rate_limit_type": "tokens_per_unit",
                                "limit_remaining": 0,
                            }
                        ],
                    }
                else:
                    return {
                        "overall_code": "OK",
                        "statuses": [
                            {
                                "code": "OK",
                                "descriptor_key": descriptor_value,
                                "rate_limit_type": "tokens_per_unit",
                                "limit_remaining": 100,
                            }
                        ],
                    }

        # Default: allow
        return {
            "overall_code": "OK",
            "statuses": [
                {
                    "code": "OK",
                    "descriptor_key": descriptor_value,
                    "rate_limit_type": "tokens_per_unit",
                    "limit_remaining": 1000,
                }
            ],
        }

    # Create 50 users: 30 premium, 20 standard
    users = []

    for i in range(30):
        user = UserAPIKeyAuth()
        user.metadata = {"priority": "premium"}
        user.user_id = f"premium_hook_user_{i}"
        users.append((user, "premium"))

    for i in range(20):
        user = UserAPIKeyAuth()
        user.metadata = {"priority": "standard"}
        user.user_id = f"standard_hook_user_{i}"
        users.append((user, "standard"))

    async def make_request(user_data):
        """Make a pre-call hook request."""
        user, priority = user_data

        try:
            result = await handler.async_pre_call_hook(
                user_api_key_dict=user,
                cache=DualCache(),
                data={"model": model},
                call_type="completion",
            )

            # If no exception, request was allowed
            successful_requests.append(
                {"user_id": user.user_id, "priority": priority, "result": "allowed"}
            )
            return {
                "status": "success",
                "user_id": user.user_id,
                "priority": priority,
            }

        except Exception as e:
            # Request was rate limited
            rate_limited_requests.append(
                {"user_id": user.user_id, "priority": priority, "error": str(e)}
            )
            return {
                "status": "rate_limited",
                "user_id": user.user_id,
                "priority": priority,
            }

    # Run all 50 requests concurrently with patches applied to the entire batch
    start_time = time.time()
    with (
        patch.object(
            handler.v3_limiter, "should_rate_limit", side_effect=mock_should_rate_limit
        ),
        patch.object(
            handler.internal_usage_cache, "async_get_cache", side_effect=mock_get_cache
        ),
    ):
        tasks = [make_request(user_data) for user_data in users]
        results = await asyncio.gather(*tasks, return_exceptions=True)
    end_time = time.time()

    # Analyze results
    successful_count = len(
        [r for r in results if isinstance(r, dict) and r["status"] == "success"]
    )
    rate_limited_count = len(
        [r for r in results if isinstance(r, dict) and r["status"] == "rate_limited"]
    )

    # Validate that premium users were mostly successful (priority worked)
    premium_results = [
        r for r in results if isinstance(r, dict) and r["priority"] == "premium"
    ]
    premium_success = len([r for r in premium_results if r["status"] == "success"])

    standard_results = [
        r for r in results if isinstance(r, dict) and r["priority"] == "standard"
    ]
    standard_success = len([r for r in standard_results if r["status"] == "success"])

    # Premium users should have higher success rate due to priority
    premium_success_rate = (
        premium_success / len(premium_results) if premium_results else 0
    )
    standard_success_rate = (
        standard_success / len(standard_results) if standard_results else 0
    )

    assert (
        premium_success_rate >= 0.9
    ), f"Premium success rate should be >= 90%, got {premium_success_rate:.2%}"
    assert (
        standard_success_rate >= 0.5
    ), f"Standard success rate should be >= 50% (with 30% random limiting, allows for variance), got {standard_success_rate:.2%}"

    # Allow for the case where both are 100% due to timing/mocking issues
    # The test is inherently flaky due to random behavior
    if premium_success_rate < 1.0 or standard_success_rate < 1.0:
        assert (
            premium_success_rate >= standard_success_rate
        ), "Premium should have >= success rate than standard"

    total_duration = end_time - start_time

    print(f"✅ Processed 50 concurrent pre-call hooks in {total_duration:.2f}s")
    print(
        f"   - Premium users: {premium_success}/{len(premium_results)} success ({premium_success_rate:.1%})"
    )
    print(
        f"   - Standard users: {standard_success}/{len(standard_results)} success ({standard_success_rate:.1%})"
    )
    print(f"   - Total successful: {successful_count}/50 ({successful_count/50:.1%})")
    print(f"   - Priority system working: Premium > Standard success rates")


# These tests make actual async_pre_call_hook calls to simulate real traffic


@pytest.mark.asyncio
async def test_fake_calls_case_1_no_rate_limiting_at_capacity(monkeypatch):
    """
    Test Case 1: Saturation-Aware Rate Limiting at 50% Threshold

    System: 100 RPM capacity, saturation_threshold=50%
    Key A: priority_reservation=0.75 (75 RPM reserved)
    Key B: priority_reservation=0.25 (25 RPM reserved)
    Traffic A: 1 request
    Traffic B: 100 requests

    Expected behavior:
    - Key A: 1 request succeeds (low traffic)
    - Key B: ~25-26 requests succeed (capped at reservation when saturation >= 50%)

    Once saturation hits 50%, strict mode enforces priority-based limits.
    """
    monkeypatch.setenv("LITELLM_LICENSE", "test-license-key")

    # Set up priority reservations
    litellm.priority_reservation = {"key_a": 0.75, "key_b": 0.25}

    dual_cache = DualCache()
    handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)

    model = "fake-call-test-1"
    total_rpm = 100

    llm_router = Router(
        model_list=[
            {
                "model_name": model,
                "litellm_params": {
                    "model": "gpt-3.5-turbo",
                    "api_key": "test-key",
                    "api_base": "test-base",
                    "rpm": total_rpm,
                },
            }
        ]
    )
    handler.update_variables(llm_router=llm_router)

    # Create users
    key_a_user = UserAPIKeyAuth()
    key_a_user.metadata = {"priority": "key_a"}
    key_a_user.user_id = "key_a_user"

    key_b_user = UserAPIKeyAuth()
    key_b_user.metadata = {"priority": "key_b"}
    key_b_user.user_id = "key_b_user"

    # Track results
    successful_requests = {"key_a": 0, "key_b": 0}
    rate_limited_requests = {"key_a": 0, "key_b": 0}

    async def make_request(user, priority_name, request_id):
        """Make a single request and track the result."""
        try:
            result = await handler.async_pre_call_hook(
                user_api_key_dict=user,
                cache=dual_cache,
                data={"model": model},
                call_type="completion",
            )

            if result is None:
                successful_requests[priority_name] += 1
                return {"status": "success", "priority": priority_name}
            else:
                rate_limited_requests[priority_name] += 1
                return {"status": "rate_limited", "priority": priority_name}

        except Exception as e:
            rate_limited_requests[priority_name] += 1
            return {
                "status": "rate_limited",
                "priority": priority_name,
                "error": str(e),
            }

    # Send 1 request from key_a, 100 from key_b
    tasks = []

    for i in range(1):
        tasks.append(make_request(key_a_user, "key_a", f"key_a_{i}"))

    for i in range(100):
        tasks.append(make_request(key_b_user, "key_b", f"key_b_{i}"))

    start_time = time.time()
    results = await asyncio.gather(*tasks, return_exceptions=True)
    end_time = time.time()

    # Analyze results
    total_successful = successful_requests["key_a"] + successful_requests["key_b"]
    total_rate_limited = rate_limited_requests["key_a"] + rate_limited_requests["key_b"]

    print(f"Test Case 1 - Saturation-Aware Rate Limiting:")
    print(f"   - Duration: {end_time - start_time:.2f}s")
    print(f"   - Key A: {successful_requests['key_a']}/1 successful (reserved 75 RPM)")
    print(
        f"   - Key B: {successful_requests['key_b']}/100 successful (reserved 25 RPM)"
    )
    print(f"   - Total successful: {total_successful}/101")
    print(f"   - Total rate limited: {total_rate_limited}/101")

    # Key A should get its 1 request
    assert (
        successful_requests["key_a"] == 1
    ), f"Key A should get 1 request, got {successful_requests['key_a']}"

    # Key B can send until saturation hits 50% (which is ~50 total requests)
    # After that, strict mode enforces its 25 RPM reservation
    # Due to race conditions in concurrent execution, allow 45-52 successful requests
    assert (
        45 <= successful_requests["key_b"] <= 52
    ), f"Key B should get ~49 requests (45-52), got {successful_requests['key_b']}"

    # Verify approximately half of key_b requests were rate limited
    assert (
        rate_limited_requests["key_b"] >= 45
    ), f"Key B should have ≥45 rate limited requests, got {rate_limited_requests['key_b']}"


@pytest.mark.asyncio
async def test_fake_calls_case_2_priority_queue_during_saturation(monkeypatch):
    """
    Test Case 2: Priority Queue Behavior During Saturation

    System: 100 RPM capacity
    Key A: priority_reservation=0.75 (75 RPM reserved)
    Key B: priority_reservation=0.25 (25 RPM reserved)
    Traffic A: 200 RPM
    Traffic B: 200 RPM
    Expected A: 75 RPM (75% of capacity)
    Expected B: 25 RPM (25% of capacity)

    When total traffic exceeds capacity, rate limiting enforces priority reservations.
    """
    monkeypatch.setenv("LITELLM_LICENSE", "test-license-key")

    litellm.priority_reservation = {"key_a": 0.75, "key_b": 0.25}

    dual_cache = DualCache()
    handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)

    model = "fake-call-test-2"
    total_rpm = 100

    llm_router = Router(
        model_list=[
            {
                "model_name": model,
                "litellm_params": {
                    "model": "gpt-3.5-turbo",
                    "api_key": "test-key",
                    "api_base": "test-base",
                    "rpm": total_rpm,
                },
            }
        ]
    )
    handler.update_variables(llm_router=llm_router)

    # Create users
    key_a_user = UserAPIKeyAuth()
    key_a_user.metadata = {"priority": "key_a"}
    key_a_user.user_id = "key_a_user"

    key_b_user = UserAPIKeyAuth()
    key_b_user.metadata = {"priority": "key_b"}
    key_b_user.user_id = "key_b_user"

    # Track results
    successful_requests = {"key_a": 0, "key_b": 0}
    rate_limited_requests = {"key_a": 0, "key_b": 0}

    async def make_request(user, priority_name, request_id):
        """Make a single request and track the result."""
        try:
            result = await handler.async_pre_call_hook(
                user_api_key_dict=user,
                cache=dual_cache,
                data={"model": model},
                call_type="completion",
            )

            if result is None:
                successful_requests[priority_name] += 1
                return {"status": "success", "priority": priority_name}
            else:
                rate_limited_requests[priority_name] += 1
                return {"status": "rate_limited", "priority": priority_name}

        except Exception as e:
            rate_limited_requests[priority_name] += 1
            return {
                "status": "rate_limited",
                "priority": priority_name,
                "error": str(e),
            }

    # Send 200 requests from each priority (over capacity)
    tasks = []

    for i in range(200):
        tasks.append(make_request(key_a_user, "key_a", f"key_a_{i}"))

    for i in range(200):
        tasks.append(make_request(key_b_user, "key_b", f"key_b_{i}"))

    start_time = time.time()
    results = await asyncio.gather(*tasks, return_exceptions=True)
    end_time = time.time()

    # Analyze results
    total_successful = successful_requests["key_a"] + successful_requests["key_b"]

    key_a_success_rate = successful_requests["key_a"] / 200
    key_b_success_rate = successful_requests["key_b"] / 200

    print(f"Test Case 2 - Priority Queue Behavior During Saturation:")
    print(f"   - Duration: {end_time - start_time:.2f}s")
    print(
        f"   - Key A: {successful_requests['key_a']}/200 successful ({key_a_success_rate:.1%})"
    )
    print(
        f"   - Key B: {successful_requests['key_b']}/200 successful ({key_b_success_rate:.1%})"
    )
    print(f"   - Total successful: {total_successful}/400")

    # Key A should get significantly more requests than Key B (75:25 ratio)
    assert (
        key_a_success_rate > key_b_success_rate
    ), f"Key A should have higher success rate: {key_a_success_rate:.1%} vs {key_b_success_rate:.1%}"

    # Check ratio is approximately 3:1 (75:25)
    if total_successful > 0:
        key_a_share = successful_requests["key_a"] / total_successful
        expected_key_a_share = 0.75

        print(
            f"   - Key A got {key_a_share:.1%} of successful requests (expected ~75%)"
        )

        # Allow tolerance for timing effects
        assert (
            abs(key_a_share - expected_key_a_share) < 0.2
        ), f"Key A share should be ~75%, got {key_a_share:.1%}"


@pytest.mark.asyncio
async def test_fake_calls_case_3_spillover_capacity_default_keys(monkeypatch):
    """
    Test Case 3: Spillover Capacity for Default Keys

    System: 100 RPM capacity
    Key A: priority_reservation=0.75 (75 RPM reserved)
    Key B: nothing set (default)
    Key C: nothing set (default)
    Key D: nothing set (default)
    Traffic A: 150 RPM
    Traffic B: 150 RPM
    Traffic C: 150 RPM
    Traffic D: 150 RPM
    Expected A: 75 RPM (75% reserved)
    Expected B: ~8.3 RPM (remaining 25 RPM / 3 default keys)
    Expected C: ~8.3 RPM
    Expected D: ~8.3 RPM

    Tests spillover behavior where default keys share remaining capacity.
    """
    monkeypatch.setenv("LITELLM_LICENSE", "test-license-key")

    litellm.priority_reservation = {"key_a": 0.75}
    litellm.priority_reservation_settings.default_priority = 0.25

    dual_cache = DualCache()
    handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)

    model = "fake-call-test-3"
    total_rpm = 100

    llm_router = Router(
        model_list=[
            {
                "model_name": model,
                "litellm_params": {
                    "model": "gpt-3.5-turbo",
                    "api_key": "test-key",
                    "api_base": "test-base",
                    "rpm": total_rpm,
                },
            }
        ]
    )
    handler.update_variables(llm_router=llm_router)

    # Create users
    key_a_user = UserAPIKeyAuth()
    key_a_user.metadata = {"priority": "key_a"}
    key_a_user.user_id = "key_a_user"

    key_b_user = UserAPIKeyAuth()
    key_b_user.metadata = {}
    key_b_user.user_id = "key_b_user"

    key_c_user = UserAPIKeyAuth()
    key_c_user.metadata = {}
    key_c_user.user_id = "key_c_user"

    key_d_user = UserAPIKeyAuth()
    key_d_user.metadata = {}
    key_d_user.user_id = "key_d_user"

    # Track results
    successful_requests = {"key_a": 0, "key_b": 0, "key_c": 0, "key_d": 0}
    rate_limited_requests = {"key_a": 0, "key_b": 0, "key_c": 0, "key_d": 0}

    async def make_request(user, key_name, request_id):
        """Make a single request and track the result."""
        try:
            result = await handler.async_pre_call_hook(
                user_api_key_dict=user,
                cache=dual_cache,
                data={"model": model},
                call_type="completion",
            )

            if result is None:
                successful_requests[key_name] += 1
                return {"status": "success", "key": key_name}
            else:
                rate_limited_requests[key_name] += 1
                return {"status": "rate_limited", "key": key_name}

        except Exception as e:
            rate_limited_requests[key_name] += 1
            return {"status": "rate_limited", "key": key_name, "error": str(e)}

    # Send 150 requests from each key (600 total, 6x over capacity)
    tasks = []

    for i in range(150):
        tasks.append(make_request(key_a_user, "key_a", f"key_a_{i}"))

    for i in range(150):
        tasks.append(make_request(key_b_user, "key_b", f"key_b_{i}"))

    for i in range(150):
        tasks.append(make_request(key_c_user, "key_c", f"key_c_{i}"))

    for i in range(150):
        tasks.append(make_request(key_d_user, "key_d", f"key_d_{i}"))

    start_time = time.time()
    results = await asyncio.gather(*tasks, return_exceptions=True)
    end_time = time.time()

    # Analyze results
    total_successful = sum(successful_requests.values())

    print(f"Test Case 3 - Spillover Capacity for Default Keys:")
    print(f"   - Duration: {end_time - start_time:.2f}s")
    print(f"   - Key A: {successful_requests['key_a']}/150 successful")
    print(f"   - Key B: {successful_requests['key_b']}/150 successful (default)")
    print(f"   - Key C: {successful_requests['key_c']}/150 successful (default)")
    print(f"   - Key D: {successful_requests['key_d']}/150 successful (default)")
    print(f"   - Total successful: {total_successful}/600")

    # Key A should get the most requests (75% of capacity)
    assert (
        successful_requests["key_a"] > successful_requests["key_b"]
    ), "Key A should get more than Key B"
    assert (
        successful_requests["key_a"] > successful_requests["key_c"]
    ), "Key A should get more than Key C"
    assert (
        successful_requests["key_a"] > successful_requests["key_d"]
    ), "Key A should get more than Key D"

    # Default keys should get similar amounts (spillover capacity)
    avg_default = (
        successful_requests["key_b"]
        + successful_requests["key_c"]
        + successful_requests["key_d"]
    ) / 3
    print(f"   - Average default key success: {avg_default:.1f}")


@pytest.mark.asyncio
async def test_fake_calls_case_4_over_allocated_with_normalization(monkeypatch):
    """
    Test Case 4: Over-Allocated Priority reservations with Normalization

    System: 100 RPM capacity
    Key A: priority_reservation=0.60 (60% requested)
    Key B: priority_reservation=0.80 (80% requested)
    Total: 140% (over-allocated, should normalize to 43%/57%)
    Traffic A: 200 RPM
    Traffic B: 200 RPM

    With saturation-aware rate limiting:
    - Initially, requests are allowed through in generous mode (under 80% saturation)
    - Once saturated, strict priority-based limits kick in with normalized weights
    - Due to concurrent burst, total successful may exceed 100 RPM in the test window
    - This test verifies normalization works and total capacity is reasonably bounded
    """
    monkeypatch.setenv("LITELLM_LICENSE", "test-license-key")

    litellm.priority_reservation = {"key_a": 0.60, "key_b": 0.80}

    dual_cache = DualCache()
    handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)

    model = "fake-call-test-4"
    total_rpm = 100

    llm_router = Router(
        model_list=[
            {
                "model_name": model,
                "litellm_params": {
                    "model": "gpt-3.5-turbo",
                    "api_key": "test-key",
                    "api_base": "test-base",
                    "rpm": total_rpm,
                },
            }
        ]
    )
    handler.update_variables(llm_router=llm_router)

    # Create users
    key_a_user = UserAPIKeyAuth()
    key_a_user.metadata = {"priority": "key_a"}
    key_a_user.user_id = "key_a_user"

    key_b_user = UserAPIKeyAuth()
    key_b_user.metadata = {"priority": "key_b"}
    key_b_user.user_id = "key_b_user"

    # Track results
    successful_requests = {"key_a": 0, "key_b": 0}
    rate_limited_requests = {"key_a": 0, "key_b": 0}

    async def make_request(user, priority_name, request_id):
        """Make a single request and track the result."""
        try:
            result = await handler.async_pre_call_hook(
                user_api_key_dict=user,
                cache=dual_cache,
                data={"model": model},
                call_type="completion",
            )

            if result is None:
                successful_requests[priority_name] += 1
                return {"status": "success", "priority": priority_name}
            else:
                rate_limited_requests[priority_name] += 1
                return {"status": "rate_limited", "priority": priority_name}

        except Exception as e:
            rate_limited_requests[priority_name] += 1
            return {
                "status": "rate_limited",
                "priority": priority_name,
                "error": str(e),
            }

    # Send 200 requests from each key (400 total, 4x over capacity)
    tasks = []

    for i in range(200):
        tasks.append(make_request(key_a_user, "key_a", f"key_a_{i}"))

    for i in range(200):
        tasks.append(make_request(key_b_user, "key_b", f"key_b_{i}"))

    start_time = time.time()
    results = await asyncio.gather(*tasks, return_exceptions=True)
    end_time = time.time()

    # Analyze results
    total_successful = successful_requests["key_a"] + successful_requests["key_b"]

    key_a_success_rate = successful_requests["key_a"] / 200
    key_b_success_rate = successful_requests["key_b"] / 200

    print(f"Test Case 4 - Over-Allocated Priority Reservations with Normalization:")
    print(f"   - Duration: {end_time - start_time:.2f}s")
    print(
        f"   - Key A (0.60): {successful_requests['key_a']}/200 successful ({key_a_success_rate:.1%})"
    )
    print(
        f"   - Key B (0.80): {successful_requests['key_b']}/200 successful ({key_b_success_rate:.1%})"
    )
    print(f"   - Total successful: {total_successful}/400")

    # With saturation-aware behavior:
    # 1. Verify total capacity is reasonably bounded (not all 400 requests succeed)
    assert (
        total_successful < 300
    ), f"Total requests should be bounded by saturation detection, got {total_successful}/400"

    # 2. Verify significant rate limiting occurred (at least 50% blocked)
    assert (
        total_successful < 200
    ), f"At least 50% of requests should be rate limited, got {total_successful}/400 successful"

    # 3. Verify both keys got some requests through (normalization is working)
    assert successful_requests["key_a"] > 0, "Key A should get some requests"
    assert successful_requests["key_b"] > 0, "Key B should get some requests"

    print(
        f"   - Normalization test PASSED: Both priorities got requests, "
        f"total bounded to {total_successful} (under 200)"
    )


@pytest.mark.asyncio
async def test_fake_calls_case_5_default_value_priority_reservation(monkeypatch):
    """
    Test Case 5: Default value for priority reservation

    System: 100 RPM capacity
    Key A: priority_reservation=0.50 (50 RPM)
    Key B: priority_reservation=0.20 (20 RPM)
    Key C: priority_reservation=0.05 (5 RPM)
    Key D: nothing set (uses default_priority=0.05, 5 RPM)
    Traffic A: 150 RPM
    Traffic B: 150 RPM
    Traffic C: 150 RPM
    Traffic D: 150 RPM
    Expected A: 55 RPM (normalized)
    Expected B: 25 RPM (normalized)
    Expected C: 10 RPM (normalized)
    Expected D: 10 RPM (normalized)

    Tests complex scenario with explicit priorities and default priority.
    """
    monkeypatch.setenv("LITELLM_LICENSE", "test-license-key")

    litellm.priority_reservation = {"key_a": 0.50, "key_b": 0.20, "key_c": 0.05}
    litellm.priority_reservation_settings.default_priority = 0.05

    dual_cache = DualCache()
    handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)

    model = "fake-call-test-5"
    total_rpm = 100

    llm_router = Router(
        model_list=[
            {
                "model_name": model,
                "litellm_params": {
                    "model": "gpt-3.5-turbo",
                    "api_key": "test-key",
                    "api_base": "test-base",
                    "rpm": total_rpm,
                },
            }
        ]
    )
    handler.update_variables(llm_router=llm_router)

    # Create users
    key_a_user = UserAPIKeyAuth()
    key_a_user.metadata = {"priority": "key_a"}
    key_a_user.user_id = "key_a_user"

    key_b_user = UserAPIKeyAuth()
    key_b_user.metadata = {"priority": "key_b"}
    key_b_user.user_id = "key_b_user"

    key_c_user = UserAPIKeyAuth()
    key_c_user.metadata = {"priority": "key_c"}
    key_c_user.user_id = "key_c_user"

    key_d_user = UserAPIKeyAuth()
    key_d_user.metadata = {}
    key_d_user.user_id = "key_d_user"

    # Track results
    successful_requests = {"key_a": 0, "key_b": 0, "key_c": 0, "key_d": 0}
    rate_limited_requests = {"key_a": 0, "key_b": 0, "key_c": 0, "key_d": 0}

    async def make_request(user, key_name, request_id):
        """Make a single request and track the result."""
        try:
            result = await handler.async_pre_call_hook(
                user_api_key_dict=user,
                cache=dual_cache,
                data={"model": model},
                call_type="completion",
            )

            if result is None:
                successful_requests[key_name] += 1
                return {"status": "success", "key": key_name}
            else:
                rate_limited_requests[key_name] += 1
                return {"status": "rate_limited", "key": key_name}

        except Exception as e:
            rate_limited_requests[key_name] += 1
            return {"status": "rate_limited", "key": key_name, "error": str(e)}

    # Send 150 requests from each key (600 total, 6x over capacity)
    tasks = []

    for i in range(150):
        tasks.append(make_request(key_a_user, "key_a", f"key_a_{i}"))

    for i in range(150):
        tasks.append(make_request(key_b_user, "key_b", f"key_b_{i}"))

    for i in range(150):
        tasks.append(make_request(key_c_user, "key_c", f"key_c_{i}"))

    for i in range(150):
        tasks.append(make_request(key_d_user, "key_d", f"key_d_{i}"))

    start_time = time.time()
    results = await asyncio.gather(*tasks, return_exceptions=True)
    end_time = time.time()

    # Analyze results
    total_successful = sum(successful_requests.values())

    print(f"Test Case 5 - Default value for priority reservation:")
    print(f"   - Duration: {end_time - start_time:.2f}s")
    print(f"   - Key A (0.50): {successful_requests['key_a']}/150 successful")
    print(f"   - Key B (0.20): {successful_requests['key_b']}/150 successful")
    print(f"   - Key C (0.05): {successful_requests['key_c']}/150 successful")
    print(f"   - Key D (default 0.05): {successful_requests['key_d']}/150 successful")
    print(f"   - Total successful: {total_successful}/600")

    # Verify priority ordering: A > B > C ≈ D
    assert (
        successful_requests["key_a"] > successful_requests["key_b"]
    ), "Key A should get more than Key B"
    assert (
        successful_requests["key_b"] > successful_requests["key_c"]
    ), "Key B should get more than Key C"

    # Key C and Key D should get similar amounts (both have 0.05 priority)
    key_c_vs_d_ratio = successful_requests["key_c"] / max(
        successful_requests["key_d"], 1
    )
    print(f"   - Key C vs Key D ratio: {key_c_vs_d_ratio:.2f} (expected ~1.0)")

    if total_successful > 0:
        key_a_share = successful_requests["key_a"] / total_successful
        print(
            f"   - Key A got {key_a_share:.1%} of successful requests (expected ~55-62%)"
        )


@pytest.mark.asyncio
async def test_default_priority_shared_pool(monkeypatch):
    """
    Test that keys without explicit priority share ONE default pool, not get individual allocations.

    With default_priority=0.25:
    - Key A, B, C (no priority) should share ONE 25 RPM pool
    - NOT get 25 RPM each (which would be 75 RPM total)
    """
    monkeypatch.setenv("LITELLM_LICENSE", "test-license-key")

    litellm.priority_reservation = {"prod": 0.75}
    litellm.priority_reservation_settings.default_priority = 0.25

    dual_cache = DualCache()
    handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)

    model = "test-default-pool"
    total_rpm = 100

    llm_router = Router(
        model_list=[
            {
                "model_name": model,
                "litellm_params": {
                    "model": "gpt-3.5-turbo",
                    "api_key": "test-key",
                    "api_base": "test-base",
                    "rpm": total_rpm,
                },
            }
        ]
    )
    handler.update_variables(llm_router=llm_router)

    # Create 3 users without explicit priority
    user_a = UserAPIKeyAuth()
    user_a.metadata = {}
    user_a.user_id = "user_a"

    user_b = UserAPIKeyAuth()
    user_b.metadata = {}
    user_b.user_id = "user_b"

    user_c = UserAPIKeyAuth()
    user_c.metadata = {}
    user_c.user_id = "user_c"

    # Get descriptors for each
    desc_a = handler._create_priority_based_descriptors(
        model=model, user_api_key_dict=user_a, priority=None
    )
    desc_b = handler._create_priority_based_descriptors(
        model=model, user_api_key_dict=user_b, priority=None
    )
    desc_c = handler._create_priority_based_descriptors(
        model=model, user_api_key_dict=user_c, priority=None
    )

    # All should use the SAME shared pool key
    assert desc_a[0]["value"] == f"{model}:default_pool"
    assert desc_b[0]["value"] == f"{model}:default_pool"
    assert desc_c[0]["value"] == f"{model}:default_pool"

    # All should have same limit (25 RPM SHARED, not 25 RPM each)
    assert desc_a[0]["rate_limit"]["requests_per_unit"] == 25
    assert desc_b[0]["rate_limit"]["requests_per_unit"] == 25
    assert desc_c[0]["rate_limit"]["requests_per_unit"] == 25

    # Verify explicit priority uses different pool
    user_prod = UserAPIKeyAuth()
    user_prod.metadata = {"priority": "prod"}
    desc_prod = handler._create_priority_based_descriptors(
        model=model, user_api_key_dict=user_prod, priority="prod"
    )

    assert desc_prod[0]["value"] == f"{model}:prod"
    assert desc_prod[0]["rate_limit"]["requests_per_unit"] == 75
    assert desc_prod[0]["value"] != desc_a[0]["value"]  # Different pools

    print("✅ Default priority test passed:")
    print(f"   - 3 keys without priority share ONE pool: {desc_a[0]['value']}")
    print(f"   - Shared pool limit: {desc_a[0]['rate_limit']['requests_per_unit']} RPM")
    print(f"   - Explicit priority 'prod' uses separate pool: {desc_prod[0]['value']}")


@pytest.mark.asyncio
async def test_async_log_success_event_increments_by_actual_tokens(monkeypatch):
    """
    Test that async_log_success_event increments token counters by actual token usage.

    This validates the fix for Bug 1: Token count was incrementing by 1 instead of actual usage.
    The async_log_success_event should increment both model_saturation_check and priority_model
    counters by the actual completion_tokens (when rate_limit_type=output).
    """
    from unittest.mock import MagicMock

    from litellm.types.utils import ModelResponse, Usage

    monkeypatch.setenv("LITELLM_LICENSE", "test-license-key")
    litellm.priority_reservation = {"dev": 0.1, "prod": 0.9}

    dual_cache = DualCache()
    handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)

    model = "test-token-increment"
    llm_router = Router(
        model_list=[
            {
                "model_name": model,
                "litellm_params": {
                    "model": "gpt-3.5-turbo",
                    "api_key": "test-key",
                    "api_base": "test-base",
                    "tpm": 1000,
                },
            }
        ]
    )
    handler.update_variables(llm_router=llm_router)

    # Track what gets incremented
    increment_calls = []

    async def mock_increment(pipeline_operations, parent_otel_span=None):
        for op in pipeline_operations:
            increment_calls.append(
                {
                    "key": op["key"],
                    "increment_value": op["increment_value"],
                }
            )

    handler.v3_limiter.async_increment_tokens_with_ttl_preservation = mock_increment

    # Create mock response with 50 completion tokens
    mock_response = MagicMock(spec=ModelResponse)
    mock_response.usage = MagicMock(spec=Usage)
    mock_response.usage.prompt_tokens = 10
    mock_response.usage.completion_tokens = 50
    mock_response.usage.total_tokens = 60

    # Create kwargs with priority in user_api_key_auth_metadata
    kwargs = {
        "standard_logging_object": {
            "metadata": {
                "user_api_key_auth_metadata": {"priority": "dev"},
            },
            "model_group": model,
        },
        "litellm_params": {
            "metadata": {"model_group": model},
        },
    }

    with patch(
        "litellm.proxy.common_utils.callback_utils.get_model_group_from_litellm_kwargs",
        return_value=model,
    ):
        await handler.async_log_success_event(
            kwargs=kwargs,
            response_obj=mock_response,
            start_time=None,
            end_time=None,
        )

    # Verify increments happened with actual token count (60 total tokens)
    assert (
        len(increment_calls) == 2
    ), f"Expected 2 increment calls, got {len(increment_calls)}"

    # Both should increment by 50 (total_tokens, since rate_limit_type defaults to 'total')
    for call in increment_calls:
        assert (
            call["increment_value"] == 60
        ), f"Expected increment of 60 tokens, got {call['increment_value']} for key {call['key']}"

    # Verify correct keys were used
    keys = [call["key"] for call in increment_calls]
    assert any(
        "model_saturation_check" in k for k in keys
    ), "Should increment model_saturation_check"
    assert any(
        "priority_model" in k and "dev" in k for k in keys
    ), "Should increment priority_model with 'dev' priority"


@pytest.mark.asyncio
async def test_saturation_check_cache_ttl_configuration(monkeypatch):
    """
    Test that saturation_check_cache_ttl controls how long saturation values are cached locally.

    This validates the configurable TTL for multi-node consistency:
    - When saturation_check_cache_ttl is set, local cache should expire after that duration
    - After expiration, fresh values should be fetched from Redis
    - This prevents nodes from having stale saturation data in multi-node deployments
    """
    monkeypatch.setenv("LITELLM_LICENSE", "test-license-key")

    # Set a short TTL for testing (5 seconds)
    original_ttl = litellm.priority_reservation_settings.saturation_check_cache_ttl
    litellm.priority_reservation_settings.saturation_check_cache_ttl = 5

    try:
        dual_cache = DualCache()
        handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)

        model = "test-saturation-ttl"
        llm_router = Router(
            model_list=[
                {
                    "model_name": model,
                    "litellm_params": {
                        "model": "gpt-3.5-turbo",
                        "api_key": "test-key",
                        "api_base": "test-base",
                        "rpm": 100,
                        "tpm": 1000,
                    },
                }
            ]
        )
        handler.update_variables(llm_router=llm_router)

        # Verify the TTL getter returns configured value
        assert (
            handler._get_saturation_check_cache_ttl() == 5
        ), "TTL should be configurable via priority_reservation_settings"

        # Track async_get_cache calls to verify TTL is passed
        get_cache_calls = []
        original_get_cache = handler.internal_usage_cache.async_get_cache

        async def mock_get_cache(
            key, litellm_parent_otel_span=None, local_only=False, **kwargs
        ):
            get_cache_calls.append(
                {
                    "key": key,
                    "ttl": kwargs.get("ttl"),
                    "local_only": local_only,
                }
            )
            return None  # Simulate cache miss

        handler.internal_usage_cache.async_get_cache = mock_get_cache

        # Call _get_saturation_value_from_cache
        counter_key = handler.v3_limiter.create_rate_limit_keys(
            key="model_saturation_check",
            value=model,
            rate_limit_type="requests",
        )

        await handler._get_saturation_value_from_cache(counter_key=counter_key)

        # Verify async_get_cache was called with the configured TTL
        assert len(get_cache_calls) == 1, "Expected 1 cache call"
        assert (
            get_cache_calls[0]["ttl"] == 5
        ), f"Expected TTL of 5 seconds, got {get_cache_calls[0]['ttl']}"
        assert (
            get_cache_calls[0]["local_only"] is False
        ), "Should check Redis (local_only=False) for multi-node consistency"

        # Test with different TTL value
        get_cache_calls.clear()
        litellm.priority_reservation_settings.saturation_check_cache_ttl = 30

        await handler._get_saturation_value_from_cache(counter_key=counter_key)

        assert (
            get_cache_calls[0]["ttl"] == 30
        ), f"TTL should update to 30 seconds, got {get_cache_calls[0]['ttl']}"

        print("Saturation check cache TTL test passed:")
        print(
            "   - TTL is configurable via priority_reservation_settings.saturation_check_cache_ttl"
        )
        print(
            "   - TTL is passed to async_get_cache for local cache expiration control"
        )
        print(
            "   - local_only=False ensures Redis is checked for multi-node consistency"
        )

    finally:
        # Restore original TTL
        litellm.priority_reservation_settings.saturation_check_cache_ttl = original_ttl


@pytest.mark.asyncio
async def test_async_log_success_event_uses_team_priority_from_auth_metadata(monkeypatch):
    """
    Test that async_log_success_event correctly retrieves priority from user_api_key_auth_metadata.

    This validates the fix where priority is retrieved from standard_logging_metadata.user_api_key_auth_metadata
    instead of just standard_logging_metadata.priority. This is important for team-based priority inheritance.
    """
    from unittest.mock import MagicMock

    from litellm.types.utils import ModelResponse, Usage

    monkeypatch.setenv("LITELLM_LICENSE", "test-license-key")
    litellm.priority_reservation = {"team_priority": 0.8, "default": 0.2}

    dual_cache = DualCache()
    handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)

    model = "test-team-priority"
    llm_router = Router(
        model_list=[
            {
                "model_name": model,
                "litellm_params": {
                    "model": "gpt-3.5-turbo",
                    "api_key": "test-key",
                    "api_base": "test-base",
                    "tpm": 1000,
                },
            }
        ]
    )
    handler.update_variables(llm_router=llm_router)

    # Track incremented keys to verify priority is used correctly
    incremented_keys = []

    async def mock_increment(pipeline_operations, parent_otel_span=None):
        for op in pipeline_operations:
            incremented_keys.append(op["key"])

    handler.v3_limiter.async_increment_tokens_with_ttl_preservation = mock_increment

    # Create mock response
    mock_response = MagicMock(spec=ModelResponse)
    mock_response.usage = MagicMock(spec=Usage)
    mock_response.usage.prompt_tokens = 10
    mock_response.usage.completion_tokens = 20
    mock_response.usage.total_tokens = 30

    # Simulate team metadata inheritance: priority is in user_api_key_auth_metadata
    # This is how the proxy passes team metadata to the callback
    kwargs = {
        "standard_logging_object": {
            "metadata": {
                # Priority NOT at top level (this would fail before the fix)
                # Priority IS in user_api_key_auth_metadata (team inheritance)
                "user_api_key_auth_metadata": {"priority": "team_priority"},
            },
            "model_group": model,
        },
        "litellm_params": {
            "metadata": {"model_group": model},
        },
    }

    with patch(
        "litellm.proxy.common_utils.callback_utils.get_model_group_from_litellm_kwargs",
        return_value=model,
    ):
        await handler.async_log_success_event(
            kwargs=kwargs,
            response_obj=mock_response,
            start_time=None,
            end_time=None,
        )

    # Verify the priority_model key uses 'team_priority' (not 'default_pool')
    priority_keys = [k for k in incremented_keys if "priority_model" in k]
    assert (
        len(priority_keys) == 1
    ), f"Expected 1 priority_model key, got {len(priority_keys)}"

    # The key should contain 'team_priority', not 'default_pool'
    assert "team_priority" in priority_keys[0], (
        f"Expected priority key to use 'team_priority' from user_api_key_auth_metadata, "
        f"got key: {priority_keys[0]}"
    )
    assert (
        "default_pool" not in priority_keys[0]
    ), f"Priority key should NOT use 'default_pool', should use team's priority. Got: {priority_keys[0]}"


@pytest.mark.asyncio
async def test_priority_429_includes_model_name_and_configured_limits(monkeypatch):
    """
    The priority-based 429 should tell operators which model was hit and what
    the model's configured TPM/RPM are, so they can decide whether to tune the
    priority allocation or the model limits.

    Regression test for the previous message that read:
        "Priority-based rate limit exceeded. Priority: prod,
         Rate limit type: tokens, Remaining: -664145,
         Model saturation: 86.3%"
    -- with no indication of which model was hit.
    """
    from fastapi import HTTPException

    monkeypatch.setenv("LITELLM_LICENSE", "test-license-key")
    litellm.priority_reservation = {"prod": 0.5}

    dual_cache = DualCache()
    handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)

    model = "gpt-4o-test"
    total_tpm = 1_000_000
    total_rpm = 10_000

    llm_router = Router(
        model_list=[
            {
                "model_name": model,
                "litellm_params": {
                    "model": "gpt-3.5-turbo",
                    "api_key": "test-key",
                    "api_base": "test-base",
                    "tpm": total_tpm,
                    "rpm": total_rpm,
                },
            }
        ]
    )
    handler.update_variables(llm_router=llm_router)

    user = UserAPIKeyAuth()
    user.metadata = {"priority": "prod"}
    user.user_id = "prod_user"

    model_group_info = handler.llm_router.get_model_group_info(model_group=model)

    # Force the atomic check+increment to return OVER_LIMIT for the
    # priority_model descriptor. saturation=0.95 keeps us above the
    # default saturation threshold so priority limits are enforced.
    over_limit_response = {
        "overall_code": "OVER_LIMIT",
        "statuses": [
            {
                "code": "OVER_LIMIT",
                "descriptor_key": "priority_model",
                "rate_limit_type": "tokens",
                "limit_remaining": -664145,
                "current_limit": int(total_tpm * 0.5),
            }
        ],
    }

    with patch.object(
        handler.v3_limiter,
        "atomic_check_and_increment_by_n",
        new=AsyncMock(return_value=over_limit_response),
    ):
        with pytest.raises(HTTPException) as exc_info:
            await handler._check_rate_limits(
                model=model,
                model_group_info=model_group_info,
                user_api_key_dict=user,
                priority="prod",
                saturation=0.95,
            )

    assert exc_info.value.status_code == 429
    detail = exc_info.value.detail
    assert isinstance(detail, dict)
    error_msg = detail["error"]

    # New fields added by this change -- the whole point of the fix.
    assert f"Model: {model}" in error_msg, error_msg
    assert f"Model TPM: {total_tpm}" in error_msg, error_msg
    assert f"Model RPM: {total_rpm}" in error_msg, error_msg

    # Existing fields must still be present (no regression).
    assert "Priority-based rate limit exceeded" in error_msg, error_msg
    assert "Priority: prod" in error_msg, error_msg
    assert "Rate limit type: tokens" in error_msg, error_msg
    assert "Model saturation:" in error_msg, error_msg


@pytest.mark.asyncio
async def test_tpm_only_model_enforces_priority_and_model_capacity(monkeypatch):
    """Regression: a model configured with ONLY tpm (no rpm) must still be
    rate limited.

    The atomic check-and-increment path used to drop any counter whose
    pre-call increment was zero. Token increments are always zero pre-call
    (usage lands on the counters post-call), so on a TPM-only model the
    limiter evaluated no counters at all: no model-wide cap, no priority
    reservation, in either mode. This test drives the real pre-call ->
    log-success -> pre-call flow with no limiter internals mocked.
    """
    from fastapi import HTTPException

    from litellm.types.utils import ModelResponse, Usage

    monkeypatch.setenv("LITELLM_LICENSE", "test-license-key")
    litellm.priority_reservation = {"dev": 0.25, "prod": 0.5}

    dual_cache = DualCache()
    handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)

    model = "tpm-only-model"
    llm_router = Router(
        model_list=[
            {
                "model_name": model,
                "litellm_params": {
                    "model": "gpt-3.5-turbo",
                    "api_key": "test-key",
                    "api_base": "test-base",
                    "tpm": 400,
                },
            }
        ]
    )
    handler.update_variables(llm_router=llm_router)

    dev_user = UserAPIKeyAuth()
    dev_user.metadata = {"priority": "dev"}
    prod_user = UserAPIKeyAuth()
    prod_user.metadata = {"priority": "prod"}

    async def record_usage(priority: str, total_tokens: int) -> None:
        await handler.async_log_success_event(
            kwargs={
                "standard_logging_object": {
                    "metadata": {"user_api_key_auth_metadata": {"priority": priority}},
                },
                "litellm_params": {"metadata": {"model_group": model}},
            },
            response_obj=ModelResponse(
                model=model,
                usage=Usage(prompt_tokens=0, completion_tokens=total_tokens, total_tokens=total_tokens),
            ),
            start_time=None,
            end_time=None,
        )

    assert (
        await handler.async_pre_call_hook(
            user_api_key_dict=dev_user, cache=dual_cache, data={"model": model}, call_type="completion"
        )
        is None
    )

    await record_usage("dev", 250)

    with pytest.raises(HTTPException) as dev_blocked:
        await handler.async_pre_call_hook(
            user_api_key_dict=dev_user, cache=dual_cache, data={"model": model}, call_type="completion"
        )
    assert dev_blocked.value.status_code == 429
    assert "Priority-based rate limit exceeded" in dev_blocked.value.detail["error"]

    assert (
        await handler.async_pre_call_hook(
            user_api_key_dict=prod_user, cache=dual_cache, data={"model": model}, call_type="completion"
        )
        is None
    )

    await record_usage("prod", 200)

    with pytest.raises(HTTPException) as capacity_blocked:
        await handler.async_pre_call_hook(
            user_api_key_dict=prod_user, cache=dual_cache, data={"model": model}, call_type="completion"
        )
    assert capacity_blocked.value.status_code == 429
    assert "Model capacity reached" in capacity_blocked.value.detail["error"]


@pytest.mark.asyncio
async def test_post_call_success_hook_attaches_priority_headers_to_dict_response():
    from litellm.proxy.hooks.parallel_request_limiter_v3 import (
        RateLimitResponse,
        RateLimitStatus,
        get_or_create_request_stash,
    )

    handler = DynamicRateLimitHandler(internal_usage_cache=DualCache())
    get_or_create_request_stash().rate_limit_response = RateLimitResponse(
        overall_code="OK",
        statuses=[
            RateLimitStatus(
                code="OK",
                current_limit=75,
                limit_remaining=74,
                rate_limit_type="requests",
                descriptor_key="priority_model",
            )
        ],
    )
    response = {
        "id": "msg_123",
        "type": "message",
        "role": "assistant",
        "content": [],
        "_hidden_params": {"additional_headers": {"x-litellm-attempted-retries": 0}},
    }

    await handler.async_post_call_success_hook(
        data={"model": "anthropic-haiku"},
        user_api_key_dict=UserAPIKeyAuth(metadata={"priority": "premium"}),
        response=response,
    )

    additional_headers = response["_hidden_params"]["additional_headers"]
    assert additional_headers["x-litellm-attempted-retries"] == 0
    assert additional_headers["x-ratelimit-priority_model-limit-requests"] == 75
    assert additional_headers["x-ratelimit-priority_model-remaining-requests"] == 74
    assert additional_headers["x-litellm-priority"] == "premium"
    assert additional_headers["x-litellm-rate-limiter-version"] == "v3"


@pytest.mark.asyncio
async def test_post_call_success_hook_leaves_raw_provider_dict_untouched():
    handler = DynamicRateLimitHandler(internal_usage_cache=DualCache())
    response = {"id": "msg_123", "type": "message", "role": "assistant", "content": []}

    await handler.async_post_call_success_hook(
        data={"model": "anthropic-haiku"},
        user_api_key_dict=UserAPIKeyAuth(metadata={"priority": "premium"}),
        response=response,
    )

    assert response == {"id": "msg_123", "type": "message", "role": "assistant", "content": []}


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("team_metadata", "expected_priority_header"),
    [
        ({"priority": "优先"}, None),
        ({"priority": "high"}, "high"),
        ({}, "default"),
    ],
)
async def test_post_call_success_hook_priority_header_is_always_http_encodable(team_metadata, expected_priority_header):
    from starlette.responses import Response

    handler = DynamicRateLimitHandler(internal_usage_cache=DualCache())
    response = {"id": "msg_123", "type": "message", "role": "assistant", "content": [], "_hidden_params": {}}

    await handler.async_post_call_success_hook(
        data={"model": "anthropic-haiku"},
        user_api_key_dict=UserAPIKeyAuth(team_id="team-1", team_metadata=team_metadata),
        response=response,
    )

    additional_headers = response["_hidden_params"]["additional_headers"]
    http_response = Response(headers={key: str(value) for key, value in additional_headers.items()})
    assert http_response.headers.get("x-litellm-priority") == expected_priority_header
    assert http_response.headers["x-litellm-rate-limiter-version"] == "v3"