The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
"""
Unit tests for CheckBatchCost class.
Covers: stale-row cleanup (file_purpose scoping), paginated find_many,
the batch_processed-column fallback query, and routing of unmanaged
Vertex (raw gs:// input_file_id) and Bedrock (raw s3:// input_file_id,
ARN unified_object_id) batches with no managed unified id.
"""
import asyncio
import json
import logging
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from copy import copy
from datetime import datetime, timezone
from hashlib import sha256
from queue import SimpleQueue
from types import SimpleNamespace
from typing import TYPE_CHECKING, Final, Literal
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import respx
from fastapi import HTTPException
from litellm.types.router import LiteLLM_Params
from litellm.types.utils import LiteLLMBatch
if TYPE_CHECKING:
from litellm.batches.batch_utils import BatchCostUsageResult
_IS_B64 = "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id"
_CLAIM_UNIFIED_BATCH_ID = "dW5pZmllZF9iYXRjaF9pZA=="
_CLAIM_OUTPUT_FILE_ID = "file-output-123"
def test_batch_output_file_object_derives_metadata():
from litellm_enterprise.proxy.hooks.managed_files import _batch_output_file_object
output_file = _batch_output_file_object(
unified_file_id="unified-output",
raw_file_id="s3://bucket/path/to/output.jsonl.out",
size_bytes=4321,
fallback=False,
)
provider_file = _batch_output_file_object(
unified_file_id="unified-provider",
raw_file_id="file-abc",
size_bytes=0,
fallback=False,
)
assert output_file.id == "unified-output"
assert output_file.object == "file"
assert output_file.purpose == "batch_output"
assert output_file.filename == "output.jsonl.out"
assert output_file.bytes == 4321
assert output_file.status == "processed"
assert provider_file.filename == "file-abc"
assert provider_file.bytes == 0
def _batch_cost_result(
cost: float,
usage: dict,
models: list[str],
successful_requests: int = 1,
failed_requests: int = 0,
) -> "BatchCostUsageResult":
"""Build the BatchCostUsageResult calculate_batch_cost_and_usage now returns,
for mocking it in tests that only care about cost/usage/models."""
from litellm.batches.batch_utils import BatchCostUsageResult
return BatchCostUsageResult(
cost=cost,
usage=usage,
models=models,
successful_requests=successful_requests,
failed_requests=failed_requests,
)
def _unmanaged_vertex_file_object(
input_file_id="gs://bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash/abc.jsonl",
status="validating",
):
"""A LiteLLMBatch JSON blob shaped like what the managed-files hook stores for an
unmanaged Vertex batch (raw gs:// input_file_id)."""
from litellm.types.utils import LiteLLMBatch
return LiteLLMBatch(
id="8823717160934178816",
completion_window="24h",
created_at=1,
endpoint="/v1/chat/completions",
input_file_id=input_file_id,
object="batch",
status=status,
).model_dump_json()
def _unmanaged_bedrock_file_object(
input_file_id=(
"s3://bucket/litellm-bedrock-files-us.anthropic.claude-sonnet-4-20250514-v1-0"
"-74b61828-9191-4d80-addb-5a0f9ab0ec6a.jsonl"
),
status="validating",
):
"""A LiteLLMBatch JSON blob shaped like what gets stored for an unmanaged Bedrock
batch (raw s3:// input_file_id, ARN unified_object_id)."""
from litellm.types.utils import LiteLLMBatch
return LiteLLMBatch(
id="arn:aws:bedrock:us-east-1:298249409318:model-invocation-job/1ofb47x17jua",
completion_window="24h",
created_at=1,
endpoint="/v1/chat/completions",
input_file_id=input_file_id,
object="batch",
status=status,
).model_dump_json()
class TestCheckBatchCost:
"""Test suite for CheckBatchCost class"""
@pytest.fixture
def mock_prisma_client(self):
client = MagicMock()
client.db = MagicMock()
client.db.litellm_managedobjecttable = MagicMock()
client.db.litellm_usertable = MagicMock()
return client
@pytest.fixture
def mock_proxy_logging_obj(self):
mock = MagicMock()
mock.get_proxy_hook.return_value = None
return mock
@pytest.fixture
def mock_llm_router(self):
return MagicMock()
@pytest.fixture
def check_batch_cost_instance(self, mock_proxy_logging_obj, mock_prisma_client, mock_llm_router):
from litellm_enterprise.proxy.common_utils.check_batch_cost import (
CheckBatchCost,
)
return CheckBatchCost(
proxy_logging_obj=mock_proxy_logging_obj,
prisma_client=mock_prisma_client,
llm_router=mock_llm_router,
)
@pytest.mark.asyncio
async def test_cleanup_scoped_to_batch_file_purpose(self, check_batch_cost_instance, mock_prisma_client):
"""_cleanup_stale_managed_objects scopes its update to file_purpose='batch' only."""
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
# Return empty so the main poll loop exits immediately
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[])
await check_batch_cost_instance.check_batch_cost()
calls = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args_list
stale_call = calls[0]
assert stale_call[1]["data"] == {"status": "stale_expired"}
where = stale_call[1]["where"]
assert where["file_purpose"] == "batch"
assert "stale_expired" in where["status"]["not_in"]
assert "created_at" in where
@pytest.mark.asyncio
async def test_startup_probe_confirms_batch_processed_support(self, check_batch_cost_instance, mock_prisma_client):
mock_prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=None)
await check_batch_cost_instance.confirm_batch_processed_support()
probe_where = mock_prisma_client.db.litellm_managedobjecttable.find_first.call_args[1]["where"]
assert probe_where["batch_processed"] is False
assert check_batch_cost_instance.batch_processed_support_confirmed is True
assert check_batch_cost_instance._has_batch_processed_column is True
@pytest.mark.asyncio
async def test_startup_probe_marks_column_absent(self, check_batch_cost_instance, mock_prisma_client):
mock_prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock(
side_effect=Exception("column batch_processed does not exist")
)
await check_batch_cost_instance.confirm_batch_processed_support()
assert check_batch_cost_instance.batch_processed_support_confirmed is False
assert check_batch_cost_instance._has_batch_processed_column is False
@pytest.mark.asyncio
async def test_startup_probe_transient_error_defers_to_poll_cycle(
self, check_batch_cost_instance, mock_prisma_client
):
mock_prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock(
side_effect=Exception("connection reset by peer")
)
await check_batch_cost_instance.confirm_batch_processed_support()
assert check_batch_cost_instance.batch_processed_support_confirmed is False
assert check_batch_cost_instance._has_batch_processed_column is True
@pytest.mark.asyncio
async def test_find_many_uses_pagination_and_excludes_stale(self, check_batch_cost_instance, mock_prisma_client):
"""find_many is called with take, order, and all terminal statuses excluded."""
from litellm.constants import MAX_OBJECTS_PER_POLL_CYCLE
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[])
await check_batch_cost_instance.check_batch_cost()
find_call = mock_prisma_client.db.litellm_managedobjecttable.find_many.call_args
assert find_call[1]["take"] == MAX_OBJECTS_PER_POLL_CYCLE
assert find_call[1]["order"] == [{"created_at": "asc"}, {"id": "asc"}]
not_in = find_call[1]["where"]["status"]["not_in"]
assert "stale_expired" in not_in
# "complete"/"completed" are intentionally NOT excluded from the
# primary query — the batch_processed=False filter is sufficient.
# This allows CheckBatchCost to pick up batches that were
# transitioned to "complete" by the retrieve_batch endpoint
# before CheckBatchCost had a chance to process them.
assert "complete" not in not_in
assert "completed" not in not_in
assert find_call[1]["where"]["batch_processed"] is False
assert check_batch_cost_instance.batch_processed_support_confirmed is True
@pytest.mark.asyncio
async def test_fallback_query_used_when_batch_processed_missing(
self, check_batch_cost_instance, mock_prisma_client
):
"""Falls back to query without batch_processed when primary query raises."""
from litellm.constants import MAX_OBJECTS_PER_POLL_CYCLE
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
# First find_many (primary query) raises with a schema error; second (fallback) returns empty
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(
side_effect=[Exception("column batch_processed does not exist"), [], []]
)
await check_batch_cost_instance.check_batch_cost()
calls = mock_prisma_client.db.litellm_managedobjecttable.find_many.call_args_list
assert len(calls) == 3
fallback_where = calls[1][1]["where"]
assert "batch_processed" not in fallback_where
assert "stale_expired" in fallback_where["status"]["not_in"]
assert calls[1][1]["take"] == MAX_OBJECTS_PER_POLL_CYCLE
# Column absence is now cached — next call should go straight to fallback
assert check_batch_cost_instance._has_batch_processed_column is False
assert check_batch_cost_instance.batch_processed_support_confirmed is False
@pytest.mark.asyncio
async def test_column_absence_cached_across_cycles(self, check_batch_cost_instance, mock_prisma_client):
"""After column absence is discovered, subsequent cycles skip the primary query entirely."""
from litellm.constants import MAX_OBJECTS_PER_POLL_CYCLE
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
# Simulate column already known absent from a previous cycle
check_batch_cost_instance._has_batch_processed_column = False
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[])
await check_batch_cost_instance.check_batch_cost()
assert mock_prisma_client.db.litellm_managedobjecttable.find_many.call_count == 2
fallback_where = mock_prisma_client.db.litellm_managedobjecttable.find_many.call_args[1]["where"]
assert "batch_processed" not in fallback_where
@pytest.mark.asyncio
async def test_fallback_completion_update_omits_batch_processed(
self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
):
"""When batch_processed column is absent, completion update must not include it.
If it did, the update would fail silently, the job would never be marked done,
and every subsequent poll cycle would re-log the cost (duplicate billing).
"""
from unittest.mock import patch
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
mock_job = MagicMock()
mock_job.id = "job-fallback-1"
mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
mock_job.created_by = "user-1"
# Simulate column already known absent (e.g. discovered on a previous cycle)
check_batch_cost_instance._has_batch_processed_column = False
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
# Build a fake batch response whose status triggers the completion branch
mock_response = MagicMock()
mock_response.status = "completed"
mock_response.output_file_id = "file-output-123"
mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"completed"}'
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})
mock_deployment = MagicMock()
mock_deployment.litellm_params.custom_llm_provider = "openai"
mock_deployment.litellm_params.model = "gpt-4"
mock_deployment.model_info.model_dump.return_value = {}
mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)
mock_file_content = MagicMock()
mock_file_content.content = b'{"id":"req-1"}'
decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;"
with (
patch(
"litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
side_effect=[decoded_id, None],
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
return_value="model-123",
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
return_value="batch-456",
),
patch(
"litellm.files.main.afile_content",
new_callable=AsyncMock,
return_value=mock_file_content,
),
patch(
"litellm.batches.batch_utils.get_file_content_as_dictionary",
return_value=[{"id": "req-1"}],
),
patch(
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
new_callable=AsyncMock,
return_value=_batch_cost_result(
0.01,
{"prompt_tokens": 10, "completion_tokens": 5},
["gpt-4"],
),
),
patch(
"litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
return_value=("gpt-4", "openai", None, None),
),
patch("litellm.litellm_core_utils.litellm_logging.Logging") as mock_logging_cls,
):
mock_logging_obj = MagicMock()
mock_logging_obj.async_success_handler = AsyncMock()
mock_logging_cls.return_value = mock_logging_obj
await check_batch_cost_instance.check_batch_cost()
# The update must have been called — this is the core assertion.
assert mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 1, (
"Expected update() to be called exactly once for the completed job"
)
update_data = mock_prisma_client.db.litellm_managedobjecttable.update.call_args[1]["data"]
assert "batch_processed" not in update_data, "update() must NOT include batch_processed when column is absent"
assert update_data["status"] == "complete"
@pytest.mark.asyncio
async def test_output_fetch_passes_deployment_credentials_as_trusted_snapshot(
self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
):
"""Bedrock resolves the output bucket ONLY from the immutable snapshot kwarg.
Spreading the credentials as plain kwargs is not enough: get_litellm_params drops
s3_bucket_name, so without _litellm_internal_model_credentials the cost poller
cannot read the output file and every completed Bedrock batch stays unbilled.
"""
from types import MappingProxyType
from unittest.mock import patch
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
mock_job = MagicMock()
mock_job.id = "job-bedrock-1"
mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
mock_job.created_by = "user-1"
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
mock_response = MagicMock()
mock_response.status = "completed"
mock_response.output_file_id = "file-output-123"
mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"completed"}'
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
mock_llm_router.get_deployment_credentials_with_provider = MagicMock(
return_value={
"custom_llm_provider": "bedrock",
"s3_bucket_name": "configured-batch-bucket",
"aws_region_name": "us-east-1",
}
)
mock_deployment = MagicMock()
mock_deployment.litellm_params.custom_llm_provider = "bedrock"
mock_deployment.litellm_params.model = "bedrock/anthropic.claude-haiku-4-5-20251001-v1:0"
mock_deployment.model_info.model_dump.return_value = {}
mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)
mock_file_content = MagicMock()
mock_file_content.content = b'{"recordId":"req-1"}'
decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;"
with (
patch(
"litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
side_effect=[decoded_id, None],
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
return_value="model-123",
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
return_value="batch-456",
),
patch(
"litellm.files.main.afile_content",
new_callable=AsyncMock,
return_value=mock_file_content,
) as mock_afile_content,
patch(
"litellm.batches.batch_utils.get_file_content_as_dictionary",
return_value=[{"recordId": "req-1"}],
),
patch(
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
new_callable=AsyncMock,
return_value=_batch_cost_result(
0.01, {"prompt_tokens": 10, "completion_tokens": 5}, ["claude-haiku-4-5"]
),
),
patch(
"litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
return_value=("anthropic.claude-haiku-4-5-20251001-v1:0", "bedrock", None, None),
),
patch("litellm.litellm_core_utils.litellm_logging.Logging") as mock_logging_cls,
):
mock_logging_obj = MagicMock()
mock_logging_obj.async_success_handler = AsyncMock()
mock_logging_cls.return_value = mock_logging_obj
await check_batch_cost_instance.check_batch_cost()
mock_afile_content.assert_awaited()
passed_kwargs = mock_afile_content.await_args[1]
snapshot = passed_kwargs.get("_litellm_internal_model_credentials")
assert snapshot is not None, "cost poller must pass the trusted credential snapshot"
assert isinstance(snapshot, MappingProxyType), (
"snapshot must be a MappingProxyType; a plain dict is rejected by get_configured_s3_bucket_name"
)
assert snapshot["s3_bucket_name"] == "configured-batch-bucket"
@pytest.mark.asyncio
async def test_poller_prices_with_deployment_registered_batch_rates(
self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
):
"""The cost poller must price with the rates the router registered for the deployment.
The deployment's raw model_info dict carries no litellm_params pricing, so passing
its model_dump() made the poller bill custom-rate batches at the public cost-map
price while the inline retrieve path billed the declared rate.
"""
from unittest.mock import patch
import litellm
deployment_id = "deploy-poller-registered-rates-1"
litellm.model_cost[deployment_id] = {
"id": deployment_id,
"input_cost_per_token_batches": 2e-06,
"output_cost_per_token_batches": 4e-06,
"litellm_provider": "bedrock",
"mode": "chat",
}
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
mock_job = MagicMock()
mock_job.id = "job-poller-rates-1"
mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
mock_job.created_by = "user-1"
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
mock_response = MagicMock()
mock_response.status = "completed"
mock_response.output_file_id = "file-output-123"
mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"completed"}'
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
mock_llm_router.get_deployment_credentials_with_provider = MagicMock(
return_value={"custom_llm_provider": "bedrock", "aws_region_name": "us-east-1"}
)
mock_deployment = MagicMock()
mock_deployment.litellm_params.custom_llm_provider = "bedrock"
mock_deployment.litellm_params.model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
mock_deployment.model_info.model_dump.return_value = {}
mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)
mock_file_content = MagicMock()
mock_file_content.content = b'{"recordId":"req-1"}'
decoded_id = f"llm_model_id,{deployment_id};llm_batch_id,batch-456;"
try:
with (
patch(
"litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
side_effect=[decoded_id, None],
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
return_value=deployment_id,
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
return_value="batch-456",
),
patch(
"litellm.files.main.afile_content",
new_callable=AsyncMock,
return_value=mock_file_content,
),
patch(
"litellm.batches.batch_utils.get_file_content_as_dictionary",
return_value=[{"recordId": "req-1"}],
),
patch(
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
new_callable=AsyncMock,
return_value=_batch_cost_result(
0.0052, {"prompt_tokens": 1400, "completion_tokens": 600}, ["claude-haiku-4-5"]
),
) as mock_calculate,
patch(
"litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
return_value=("us.anthropic.claude-haiku-4-5-20251001-v1:0", "bedrock", None, None),
),
patch("litellm.litellm_core_utils.litellm_logging.Logging") as mock_logging_cls,
):
mock_logging_obj = MagicMock()
mock_logging_obj.async_success_handler = AsyncMock()
mock_logging_cls.return_value = mock_logging_obj
await check_batch_cost_instance.check_batch_cost()
finally:
litellm.model_cost.pop(deployment_id, None)
mock_calculate.assert_awaited_once()
passed_model_info = mock_calculate.await_args.kwargs["model_info"]
assert passed_model_info is not None, "poller must pass the deployment's registered pricing"
assert passed_model_info["input_cost_per_token_batches"] == 2e-06
assert passed_model_info["output_cost_per_token_batches"] == 4e-06
@pytest.mark.asyncio
async def test_poller_masks_api_base_credentials_before_logging(
self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
):
"""Request rows mask `key=` query credentials out of api_base before it is
logged, but the poller skips that pre-call step, so an unmasked deployment
api_base would land verbatim on the batch cost row: regression test for the
poller masking the same way.
"""
import base64
from unittest.mock import patch
import httpx
import respx
from litellm.litellm_core_utils.litellm_logging import Logging
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
mock_job = MagicMock()
mock_job.id = "job-masked-api-base-1"
mock_job.unified_object_id = base64.urlsafe_b64encode(
b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
).decode()
mock_job.created_by = "user-1"
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
mock_response = MagicMock()
mock_response.status = "completed"
mock_response.output_file_id = "file-output-123"
mock_response.error_file_id = None
mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"completed"}'
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})
mock_deployment = MagicMock()
mock_deployment.litellm_params.custom_llm_provider = "openai"
mock_deployment.litellm_params.model = "gpt-5.4-mini"
mock_deployment.litellm_params.api_base = "https://gateway.example.com/v1?key=AIzaSyVERYSECRET7890"
mock_deployment.model_info.model_dump.return_value = {}
mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)
output_line = json.dumps(
{
"custom_id": "req-1",
"response": {
"status_code": 200,
"body": {
"id": "chatcmpl-1",
"object": "chat.completion",
"model": "gpt-5.4-mini",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "hi"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
},
},
"error": None,
}
)
with (
respx.mock(assert_all_called=True) as provider,
patch.object( # test-quality-ok: the poller builds Logging inline, the only seam to the row it logs
Logging, "async_success_handler", autospec=True
) as success_handler,
):
provider.get("https://api.openai.com/v1/files/file-output-123/content").mock(
return_value=httpx.Response(200, content=f"{output_line}\n".encode())
)
await check_batch_cost_instance.check_batch_cost()
cost_row_calls = [call for call in success_handler.await_args_list if "batch_cost" in call.kwargs]
assert len(cost_row_calls) == 1
logged_api_base = cost_row_calls[0].args[0].litellm_params["api_base"]
assert logged_api_base == "https://gateway.example.com/v1?key=*****7890"
assert "VERYSECRET" not in logged_api_base
@pytest.mark.asyncio
async def test_primary_path_completion_update_includes_batch_processed(
self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
):
"""When batch_processed column IS present, completion update must set it to True.
This is the symmetric counterpart to test_fallback_completion_update_omits_batch_processed
and proves the conditional on _has_batch_processed_column governs the update data.
"""
from unittest.mock import patch
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
mock_job = MagicMock()
mock_job.id = "job-primary-1"
mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
mock_job.created_by = "user-1"
assert check_batch_cost_instance._has_batch_processed_column is True
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
mock_response: Final = LiteLLMBatch(
id="batch-1",
status="completed",
output_file_id="file-output-123",
created_at=1,
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id="file-input-1",
object="batch",
)
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})
mock_deployment = MagicMock()
mock_deployment.litellm_params.custom_llm_provider = "openai"
mock_deployment.litellm_params.model = "gpt-4"
mock_deployment.model_info.model_dump.return_value = {}
mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)
mock_file_content = MagicMock()
mock_file_content.content = b'{"id":"req-1"}'
decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;"
with (
patch(
"litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
side_effect=[decoded_id, None],
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
return_value="model-123",
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
return_value="batch-456",
),
patch(
"litellm.files.main.afile_content",
new_callable=AsyncMock,
return_value=mock_file_content,
),
patch(
"litellm.batches.batch_utils.get_file_content_as_dictionary",
return_value=[{"id": "req-1"}],
),
patch(
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
new_callable=AsyncMock,
return_value=_batch_cost_result(
0.01,
{"prompt_tokens": 10, "completion_tokens": 5},
["gpt-4"],
),
),
patch(
"litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
return_value=("gpt-4", "openai", None, None),
),
patch("litellm.litellm_core_utils.litellm_logging.Logging") as mock_logging_cls,
):
mock_logging_obj = MagicMock()
mock_logging_obj.async_success_handler = AsyncMock()
mock_logging_cls.return_value = mock_logging_obj
await check_batch_cost_instance.check_batch_cost()
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
"Expected update() to be called exactly once for the completed job"
)
update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
assert update_data["batch_processed"] is True, (
"update() must include batch_processed=True when column is present"
)
assert update_data["status"] == "complete"
@pytest.mark.asyncio
async def test_completed_batch_with_no_attributable_owner_still_writes_spend_log(
self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
):
"""Regression: a batch created with the master key or a team-less key has
created_by=None and team_id=None on LiteLLM_ManagedObjectTable (the table
never stores the raw key hash). CheckBatchCost's synthetic logging_obj for
such a batch then carries no attributable key/user/team/end-user, and
before the fix _should_track_cost_callback silently skipped the DB write
with no error or warning: batch_processed still became True, but no
LiteLLM_SpendLogs row was ever written.
Unlike the other tests in this file, this one does NOT mock
litellm_logging.Logging or async_success_handler -- it runs the real
logging pipeline through to _ProxyDBLogger, which is the exact gap that
let the original bug ship undetected.
"""
import litellm
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
mock_job = MagicMock()
mock_job.id = "job-unattributed-1"
mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
mock_job.created_by = None
mock_job.team_id = None
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
# A real LiteLLMBatch (not a bare MagicMock): this test runs the real
# litellm_logging.Logging pipeline, which type-checks the result via
# isinstance(..., LiteLLMBatch) before it will compute/attach a cost.
from litellm.types.utils import LiteLLMBatch
mock_response = LiteLLMBatch(
id="batch-1",
completion_window="24h",
created_at=1,
endpoint="/v1/chat/completions",
input_file_id="file-input-123",
object="batch",
status="completed",
output_file_id="file-output-123",
)
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})
mock_deployment = MagicMock()
mock_deployment.litellm_params.custom_llm_provider = "openai"
mock_deployment.litellm_params.model = "gpt-4"
mock_deployment.model_info.model_dump.return_value = {}
mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)
mock_file_content = MagicMock()
mock_file_content.content = b'{"id":"req-1"}'
decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;"
db_logger = ProxyDBLogger()
mock_update_database = AsyncMock()
# Unlike the other tests in this file, this one runs the real
# litellm_logging.Logging pipeline, which calls
# _is_base64_encoded_unified_file_id an extra time (checking result.id
# after it's reset to job.unified_object_id). Key off the argument
# instead of a fixed-length side_effect list so the exact call count
# doesn't matter.
def _fake_is_base64_encoded(file_id):
return decoded_id if file_id == mock_job.unified_object_id else None
with (
patch(
"litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
side_effect=_fake_is_base64_encoded,
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
return_value="model-123",
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
return_value="batch-456",
),
patch(
"litellm.files.main.afile_content",
new_callable=AsyncMock,
return_value=mock_file_content,
),
patch(
"litellm.batches.batch_utils.get_file_content_as_dictionary",
return_value=[{"id": "req-1"}],
),
patch(
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
new_callable=AsyncMock,
return_value=_batch_cost_result(
0.01,
{"prompt_tokens": 10, "completion_tokens": 5},
["gpt-4"],
),
),
patch(
"litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
return_value=("gpt-4", "openai", None, None),
),
patch.object(litellm, "_async_success_callback", [db_logger]),
patch(
"litellm.proxy.proxy_server.proxy_logging_obj",
MagicMock(
db_spend_update_writer=MagicMock(update_database=mock_update_database),
slack_alerting_instance=MagicMock(customer_spend_alert=AsyncMock()),
),
),
patch("litellm.proxy.proxy_server.increment_spend_counters", AsyncMock()),
patch("litellm.proxy.proxy_server.update_cache", AsyncMock()),
):
await check_batch_cost_instance.check_batch_cost()
mock_update_database.assert_awaited_once()
assert mock_update_database.call_args.kwargs["response_cost"] == 0.01
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
"the job must still be marked processed once cost tracking succeeds"
)
@pytest.mark.asyncio
async def test_cost_tracking_failure_leaves_job_unprocessed(
self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
):
"""LIT-4008 regression: when fetching a completed batch's results fails
(e.g. Anthropic rejecting a msgbatch_ id on the Files API), the job must
NOT be marked complete/batch_processed. Pre-fix the $0 spend row was
written and batch_processed=True made it permanent; the failure must
instead leave the row untouched so the next poll retries, without
aborting the poll cycle.
"""
from unittest.mock import patch
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
mock_job = MagicMock()
mock_job.id = "job-anthropic-1"
mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
mock_job.created_by = "user-1"
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
mock_response = MagicMock()
mock_response.status = "completed"
mock_response.output_file_id = "msgbatch_01WA5hdsa2Xx8w4zyPjV1frs"
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
mock_llm_router.get_deployment_credentials_with_provider = MagicMock(
return_value={"api_key": "sk-test", "custom_llm_provider": "anthropic"}
)
decoded_id = "llm_model_id,model-123;llm_batch_id,msgbatch_01WA5hdsa2Xx8w4zyPjV1frs;"
with (
patch(
"litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
side_effect=[decoded_id, None],
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
return_value="model-123",
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
return_value="msgbatch_01WA5hdsa2Xx8w4zyPjV1frs",
),
patch(
"litellm.files.main.afile_content",
new_callable=AsyncMock,
side_effect=Exception("File id must have `file_` prefix."),
),
):
await check_batch_cost_instance.check_batch_cost()
assert mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 0, (
"a failed cost tracking attempt must not mark the job processed"
)
@pytest.mark.asyncio
@pytest.mark.parametrize("terminal_status", ["failed", "expired", "cancelled"])
async def test_terminal_status_marks_job_processed(
self,
check_batch_cost_instance,
mock_prisma_client,
mock_llm_router,
terminal_status,
):
"""When the provider reports a terminal status with nothing to bill
(failed/cancelled, or expired with no output file), the row must be written back
with that status and batch_processed=True so it stops being polled forever.
"""
import base64
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
mock_job = MagicMock()
mock_job.id = "job-terminal-1"
mock_job.unified_object_id = base64.urlsafe_b64encode(
b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
).decode()
mock_job.created_by = "user-1"
assert check_batch_cost_instance._has_batch_processed_column is True
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
mock_response = MagicMock()
mock_response.status = terminal_status
mock_response.output_file_id = None
mock_response.model_dump_json.return_value = f'{{"id":"batch-1","status":"{terminal_status}"}}'
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
await check_batch_cost_instance.check_batch_cost()
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
f"Expected update() to be called exactly once for a {terminal_status} job"
)
update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
assert update_data["status"] == terminal_status
assert update_data["batch_processed"] is True, (
"terminal-status update() must set batch_processed=True so polling stops"
)
@pytest.mark.asyncio
@pytest.mark.parametrize("terminal_status", ["failed", "cancelled"])
async def test_terminal_status_persists_managed_output_file_ids(
self,
check_batch_cost_instance,
mock_prisma_client,
mock_llm_router,
terminal_status,
):
"""A cancelled/failed batch with a provider error file (and no output file) must
be persisted with unified managed file IDs, never raw provider IDs. Raw IDs
written here leak to every later GET /batches/{id} and GET /batches because the
terminal row is final (batch_processed=True) and read paths only resolve, never
mint. (Any terminal status with an output file is billed through the completed
path instead, covered by test_terminal_status_with_output_file_is_billed.)
"""
import base64
import json
from litellm.types.utils import LiteLLMBatch
unified_batch_uid = base64.urlsafe_b64encode(
b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
).decode()
raw_error_file_id = "file-terminal-err-xyz"
raw_input_file_id = "file-terminal-in-123"
unified_input_file_id = base64.urlsafe_b64encode(
b"litellm_proxy:application/octet-stream;unified_id,in-1;target_model_names,gpt-5-batch"
).decode()
unified_error_file_id = base64.urlsafe_b64encode(
f"litellm_proxy:application/octet-stream;unified_id,u-2;llm_output_file_id,{raw_error_file_id}".encode()
).decode()
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
input_file_row = MagicMock()
input_file_row.unified_file_id = unified_input_file_id
def find_managed_file(where):
if where["flat_model_file_ids"]["has"] == raw_input_file_id:
return input_file_row
return None
mock_prisma_client.db.litellm_managedfiletable.find_first = AsyncMock(side_effect=find_managed_file)
mock_job = MagicMock()
mock_job.id = "job-terminal-mint-1"
mock_job.unified_object_id = unified_batch_uid
mock_job.created_by = "user-1"
mock_job.team_id = "team-1"
check_batch_cost_instance._has_batch_processed_column = True
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
response = LiteLLMBatch(
id="batch-456",
completion_window="24h",
created_at=1,
endpoint="/v1/chat/completions",
input_file_id=raw_input_file_id,
object="batch",
status=terminal_status,
output_file_id=None,
error_file_id=raw_error_file_id,
)
mock_llm_router.aretrieve_batch = AsyncMock(return_value=response)
from litellm.proxy.openai_files_endpoints.common_utils import ManagedBatchOutputFileWriter
mock_hook = MagicMock(spec=ManagedBatchOutputFileWriter)
mock_hook.get_unified_output_file_id.side_effect = [unified_error_file_id]
mock_hook.store_batch_output_file = AsyncMock()
check_batch_cost_instance.proxy_logging_obj.get_proxy_hook.return_value = mock_hook
await check_batch_cost_instance.check_batch_cost()
mock_hook.get_unified_output_file_id.assert_called_once_with(
output_file_id=raw_error_file_id,
model_id="model-123",
model_name="gpt-5-batch",
)
mock_hook.store_batch_output_file.assert_awaited_once_with(
unified_file_id=unified_error_file_id,
provider_file_id=raw_error_file_id,
model_id="model-123",
model_name="gpt-5-batch",
owner=mock_hook.store_batch_output_file.await_args.kwargs["owner"],
litellm_parent_otel_span=None,
size_bytes=None,
fetch_provider_details=True,
)
owner = mock_hook.store_batch_output_file.await_args.kwargs["owner"]
assert owner.user_id == "user-1"
assert owner.team_id == "team-1"
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2
update_call = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args
assert update_call.kwargs["where"] == {
"id": "job-terminal-mint-1",
"batch_processed": False,
"updated_at": mock_job.updated_at,
}
update_data = update_call.kwargs["data"]
assert update_data["status"] == terminal_status
assert update_data["batch_processed"] is True
persisted = json.loads(update_data["file_object"])
assert persisted["id"] == unified_batch_uid
assert persisted["input_file_id"] == unified_input_file_id
assert persisted["output_file_id"] is None
assert persisted["error_file_id"] == unified_error_file_id
assert raw_error_file_id not in update_data["file_object"]
@pytest.mark.asyncio
@pytest.mark.parametrize("completed_status", ["completed", "complete"])
async def test_completed_without_output_file_marked_processed_without_billing(
self,
check_batch_cost_instance,
mock_prisma_client,
mock_llm_router,
completed_status,
):
"""#35354 regression: a terminal completed batch whose request lines all failed
reaches `completed` with output_file_id=None (only an error_file_id).
Pre-fix it matched neither the completed-with-output branch nor the
failed/expired/cancelled branch, so batch_processed stayed False and the row
was re-selected on every poll cycle forever. It must now be marked terminal
exactly once, without being billed: request_counts.completed == 0 proves the
missing output file means nothing to bill rather than a lagging output id
(#37713 keeps the lagging case eligible for the next cycle).
"""
import base64
from unittest.mock import patch
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
mock_job = MagicMock()
mock_job.id = "job-completed-no-output-1"
mock_job.unified_object_id = base64.urlsafe_b64encode(
b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
).decode()
mock_job.created_by = "user-1"
assert check_batch_cost_instance._has_batch_processed_column is True
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
mock_response = MagicMock()
mock_response.status = completed_status
mock_response.output_file_id = None
mock_response.error_file_id = "file-error-123"
mock_response.request_counts = MagicMock(completed=0, failed=3, total=3)
mock_response.model_dump_json.return_value = f'{{"id":"batch-1","status":"{completed_status}"}}'
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
# Billing reads credentials off the router; if it is touched we billed a batch
# that has no output, which is the behaviour this test guards against.
mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})
with patch(
"litellm.files.main.afile_content",
new_callable=AsyncMock,
) as mock_afile_content:
await check_batch_cost_instance.check_batch_cost()
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
"a completed batch with no output file must be marked processed exactly once"
)
update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
assert update_data["status"] == completed_status
assert update_data["batch_processed"] is True, (
"completed-without-output update() must set batch_processed=True so polling stops"
)
assert mock_afile_content.await_count == 0, "a batch with no output file must not be billed"
assert mock_llm_router.get_deployment_credentials_with_provider.call_count == 0, (
"a batch with no output file must not enter the cost-tracking path"
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"request_counts",
[
MagicMock(completed=7, failed=0, total=7),
None,
MagicMock(completed=0, failed=0, total=0),
],
ids=["lagging_output_id", "unknown_counts", "synthesized_zero_counts"],
)
async def test_completed_with_lagging_output_file_left_for_next_cycle(
self,
check_batch_cost_instance,
mock_prisma_client,
mock_llm_router,
request_counts,
):
"""#37713 regression: a batch can report completed while its output_file_id is
still lagging behind at the provider. Retiring it in that window (or when the
request counts cannot prove there is nothing to bill) permanently loses the
spend record, so the poller must leave the row untouched and revisit it on the
next cycle once the output id has appeared.
"""
import base64
from unittest.mock import patch
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
mock_job = MagicMock()
mock_job.id = "job-completed-lagging-output-1"
mock_job.unified_object_id = base64.urlsafe_b64encode(
b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
).decode()
mock_job.created_by = "user-1"
assert check_batch_cost_instance._has_batch_processed_column is True
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
mock_response = MagicMock()
mock_response.status = "completed"
mock_response.output_file_id = None
mock_response.error_file_id = None
mock_response.request_counts = request_counts
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})
with patch(
"litellm.files.main.afile_content",
new_callable=AsyncMock,
) as mock_afile_content:
await check_batch_cost_instance.check_batch_cost()
assert mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 0, (
"a completed batch whose output id is still lagging must stay eligible for the next poll"
)
assert mock_afile_content.await_count == 0, "a batch with no output file must not be billed"
@pytest.mark.asyncio
async def test_non_terminal_status_left_unprocessed(
self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
):
"""A batch still validating/in_progress must NOT be treated as terminal: no DB
write, so it keeps being polled until it actually reaches a terminal status.
"""
from unittest.mock import patch
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
mock_job = MagicMock()
mock_job.id = "job-in-progress-1"
mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
mock_job.created_by = "user-1"
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
mock_response = MagicMock()
mock_response.status = "in_progress"
mock_response.output_file_id = None
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;"
with (
patch(
"litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
side_effect=[decoded_id, None],
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
return_value="model-123",
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
return_value="batch-456",
),
):
await check_batch_cost_instance.check_batch_cost()
assert mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 0, (
"a non-terminal batch must not be written back (would stop polling prematurely)"
)
@pytest.mark.asyncio
@pytest.mark.parametrize("terminal_status", ["expired", "cancelled", "failed"])
async def test_terminal_status_with_output_file_is_billed(
self,
check_batch_cost_instance,
mock_prisma_client,
mock_llm_router,
terminal_status,
):
"""A terminal (expired/cancelled/failed) batch that still produced an output file
served real request lines, so it must be billed (cost tracked) and then marked
processed, not silently marked terminal without billing.
"""
from unittest.mock import patch
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
mock_job = MagicMock()
mock_job.id = "job-terminal-with-output-1"
mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
mock_job.created_by = "user-1"
assert check_batch_cost_instance._has_batch_processed_column is True
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
mock_response: Final = LiteLLMBatch(
id="batch-1",
status=terminal_status,
output_file_id="file-output-123",
created_at=1,
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id="file-input-1",
object="batch",
)
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})
mock_deployment = MagicMock()
mock_deployment.litellm_params.custom_llm_provider = "openai"
mock_deployment.litellm_params.model = "gpt-4"
mock_deployment.model_info.model_dump.return_value = {}
mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)
mock_file_content = MagicMock()
mock_file_content.content = b'{"id":"req-1"}'
decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;"
with (
patch(
"litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
side_effect=[decoded_id, None],
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
return_value="model-123",
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
return_value="batch-456",
),
patch(
"litellm.files.main.afile_content",
new_callable=AsyncMock,
return_value=mock_file_content,
) as mock_afile_content,
patch(
"litellm.batches.batch_utils.get_file_content_as_dictionary",
return_value=[{"id": "req-1"}],
),
patch(
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
new_callable=AsyncMock,
return_value=_batch_cost_result(
0.01,
{"prompt_tokens": 10, "completion_tokens": 5},
["gpt-4"],
),
),
patch(
"litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
return_value=("gpt-4", "openai", None, None),
),
patch("litellm.litellm_core_utils.litellm_logging.Logging") as mock_logging_cls,
):
mock_logging_obj = MagicMock()
mock_logging_obj.async_success_handler = AsyncMock()
mock_logging_cls.return_value = mock_logging_obj
await check_batch_cost_instance.check_batch_cost()
assert mock_afile_content.await_count == 1, (
f"{terminal_status} batch with an output file must fetch results and be billed"
)
mock_logging_obj.async_success_handler.assert_awaited_once()
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2
update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
assert update_data["batch_processed"] is True
assert update_data["status"] == terminal_status, (
f"billed {terminal_status} batch must keep its real terminal status in the DB"
)
@pytest.mark.asyncio
async def test_error_file_failures_add_to_failed_request_count(
self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
):
"""OpenAI-shaped providers report per-request failures only in a separate
error file. The poller prices from the output file, so without also counting
the error file's lines, batch_failed_requests on the spend log undercounts:
regression test for the poller path merging error-file failures.
"""
import base64
from unittest.mock import patch
import httpx
import respx
from litellm.litellm_core_utils.litellm_logging import Logging
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
mock_job = MagicMock()
mock_job.id = "job-error-file-1"
mock_job.unified_object_id = base64.urlsafe_b64encode(
b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
).decode()
mock_job.created_by = "user-1"
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
mock_response = MagicMock()
mock_response.status = "completed"
mock_response.output_file_id = "file-output-123"
mock_response.error_file_id = "file-error-456"
mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"completed"}'
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})
mock_deployment = MagicMock()
mock_deployment.litellm_params.custom_llm_provider = "openai"
mock_deployment.litellm_params.model = "gpt-4"
mock_deployment.model_info.model_dump.return_value = {}
mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)
succeeded_line = json.dumps(
{
"custom_id": "req-1",
"response": {
"status_code": 200,
"body": {
"id": "chatcmpl-1",
"object": "chat.completion",
"model": "gpt-4",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "hi"},
"finish_reason": "stop",
}
],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 5,
"total_tokens": 15,
},
},
},
"error": None,
}
)
rejected_line = json.dumps(
{
"custom_id": "req-2",
"response": {
"status_code": 400,
"body": {"error": {"message": "bad request"}},
},
"error": None,
}
)
error_file_lines = "\n".join(
json.dumps({"custom_id": custom_id, "error": {"message": "rejected"}}) for custom_id in ("req-3", "req-4")
)
with (
respx.mock(assert_all_called=True) as provider,
patch.object( # test-quality-ok: the poller builds Logging inline, the only seam to its handler kwargs
Logging, "async_success_handler", new_callable=AsyncMock
) as success_handler,
):
provider.get("https://api.openai.com/v1/files/file-output-123/content").mock(
return_value=httpx.Response(200, content=f"{succeeded_line}\n{rejected_line}\n".encode())
)
provider.get("https://api.openai.com/v1/files/file-error-456/content").mock(
return_value=httpx.Response(200, content=f"{error_file_lines}\n\n".encode())
)
await check_batch_cost_instance.check_batch_cost()
spend_log_calls = [call.kwargs for call in success_handler.await_args_list if "batch_cost" in call.kwargs]
assert len(spend_log_calls) == 1
handler_kwargs = spend_log_calls[0]
assert handler_kwargs["batch_successful_requests"] == 1
assert handler_kwargs["batch_failed_requests"] == 3, (
"2 error-file lines must add to the output file's 1 rejected request"
)
assert handler_kwargs["batch_models"] == ["gpt-4"]
assert handler_kwargs["batch_usage"].total_tokens == 15
assert handler_kwargs["batch_cost"] > 0
@pytest.mark.asyncio
async def test_terminal_batch_with_missing_output_file_is_retired_unbilled(
self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
):
"""A terminal batch whose advertised output file 404s at the provider has
nothing to fetch on this or any later poll (Vertex AI advertises an output
path for every batch, even ones that never wrote it), so the job must be
retired as terminal on the first cycle instead of retrying until the
staleness sweep gives up on it.
"""
import base64
from unittest.mock import patch
from litellm.exceptions import NotFoundError
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
mock_job = MagicMock()
mock_job.id = "job-output-gone-1"
mock_job.unified_object_id = base64.urlsafe_b64encode(
b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
).decode()
mock_job.created_by = "user-1"
assert check_batch_cost_instance._has_batch_processed_column is True
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
missing_output_file_id = "gs://batch-out/job-1/predictions.jsonl"
mock_response = MagicMock()
mock_response.status = "failed"
mock_response.output_file_id = missing_output_file_id
mock_response.error_file_id = None
mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"failed"}'
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})
with (
patch(
"litellm.files.main.afile_content",
new_callable=AsyncMock,
side_effect=NotFoundError(
message=f"404: output file {missing_output_file_id} does not exist",
model="gemini-2.5-pro",
llm_provider="vertex_ai",
),
) as mock_afile_content,
patch(
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
new_callable=AsyncMock,
) as mock_calculate,
):
await check_batch_cost_instance.check_batch_cost()
assert mock_afile_content.await_count == 1
mock_calculate.assert_not_awaited()
assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
"a terminal batch with a 404ing output file must be retired, not retried forever"
)
update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
assert update_data["status"] == "failed"
assert update_data["batch_processed"] is True
@pytest.mark.asyncio
async def test_raw_output_file_id_converted_to_managed_id(
self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
):
"""CheckBatchCost must convert a raw provider output_file_id to a managed base64 ID.
Without this, GET /batches/{id} returns a raw file ID that cannot be routed
through the proxy, causing API_KEY errors when clients call GET /files/{id}/content.
"""
from litellm.proxy.openai_files_endpoints.common_utils import ManagedBatchOutputFileWriter
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
mock_job = MagicMock()
mock_job.id = "job-raw-file-1"
mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
mock_job.created_by = "user-1"
mock_job.team_id = None
check_batch_cost_instance._has_batch_processed_column = True
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
raw_output_file_id = "file-batch-output-abc123"
raw_error_file_id = "file-batch-error-xyz456"
fake_managed_output_id = "bGl0ZWxsbV9wcm94eTo6b3V0cHV0"
fake_managed_error_id = "bGl0ZWxsbV9wcm94eTo6ZXJyb3I="
mock_response = MagicMock()
mock_response.status = "completed"
mock_response.output_file_id = raw_output_file_id
mock_response.error_file_id = raw_error_file_id
mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"completed"}'
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})
mock_deployment = MagicMock()
mock_deployment.litellm_params.custom_llm_provider = "azure"
mock_deployment.litellm_params.model = "azure/gpt-5-mini"
mock_deployment.model_name = "gpt-5-batch"
mock_deployment.model_info.model_dump.return_value = {}
mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)
mock_hook = MagicMock(spec=ManagedBatchOutputFileWriter)
mock_hook.get_unified_output_file_id.side_effect = [
fake_managed_output_id,
fake_managed_error_id,
]
mock_hook.store_batch_output_file = AsyncMock()
check_batch_cost_instance.proxy_logging_obj.get_proxy_hook.return_value = mock_hook
mock_file_content = MagicMock()
mock_file_content.content = b'{"id":"req-1"}'
decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;"
with (
patch(
"litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
# call 1: job unified_object_id decode, call 2: existing raw check for output_file_id,
# call 3: fix guard for output_file_id, call 4: fix guard for error_file_id
side_effect=[decoded_id, None, None, None],
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
return_value="model-123",
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
return_value="batch-456",
),
patch(
"litellm.files.main.afile_content",
new_callable=AsyncMock,
return_value=mock_file_content,
),
patch(
"litellm.batches.batch_utils.get_file_content_as_dictionary",
return_value=[{"id": "req-1"}],
),
patch(
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
new_callable=AsyncMock,
return_value=_batch_cost_result(
0.01,
{"prompt_tokens": 10, "completion_tokens": 5},
["gpt-4"],
),
),
patch(
"litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
return_value=("gpt-5-mini", "azure", None, None),
),
patch("litellm.litellm_core_utils.litellm_logging.Logging") as mock_logging_cls,
):
mock_logging_obj = MagicMock()
mock_logging_obj.async_success_handler = AsyncMock()
mock_logging_cls.return_value = mock_logging_obj
await check_batch_cost_instance.check_batch_cost()
assert mock_hook.get_unified_output_file_id.call_count == 2
mock_hook.get_unified_output_file_id.assert_any_call(
output_file_id=raw_output_file_id,
model_id="model-123",
model_name="gpt-5-batch",
)
mock_hook.get_unified_output_file_id.assert_any_call(
output_file_id=raw_error_file_id,
model_id="model-123",
model_name="gpt-5-batch",
)
assert mock_hook.store_batch_output_file.await_count == 2
assert {call.kwargs["model_name"] for call in mock_hook.store_batch_output_file.await_args_list} == {
"gpt-5-batch"
}
stored = {
c.kwargs["provider_file_id"]: c.kwargs["unified_file_id"]
for c in mock_hook.store_batch_output_file.call_args_list
}
assert stored == {
raw_output_file_id: fake_managed_output_id,
raw_error_file_id: fake_managed_error_id,
}
stored_sizes = {
c.kwargs["provider_file_id"]: c.kwargs["size_bytes"]
for c in mock_hook.store_batch_output_file.call_args_list
}
assert stored_sizes[raw_output_file_id] == len(mock_file_content.content)
assert stored_sizes[raw_error_file_id] is None
assert mock_response.output_file_id == fake_managed_output_id
assert mock_response.error_file_id == fake_managed_error_id
class TestUnmanagedVertexRouting:
"""Routing of unmanaged Vertex batches whose unified_object_id is a raw provider job id."""
def _instance(self, track_unmanaged, router):
from litellm_enterprise.proxy.common_utils.check_batch_cost import (
CheckBatchCost,
)
return CheckBatchCost(
proxy_logging_obj=MagicMock(),
prisma_client=MagicMock(),
llm_router=router,
track_unmanaged_batch_cost=track_unmanaged,
)
def _job(self, file_object=None):
job = MagicMock()
job.unified_object_id = "8823717160934178816"
job.file_object = file_object if file_object is not None else _unmanaged_vertex_file_object()
return job
def test_flag_off_skips_unmanaged_id_unchanged(self):
"""Default (flag off): a raw numeric unified_object_id is skipped exactly as before;
no model derivation or router lookup happens."""
router = MagicMock()
instance = self._instance(track_unmanaged=False, router=router)
prom = MagicMock()
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(self._job(), prom)
assert result is None
prom.record_check_batch_cost_error.assert_called_once_with("invalid_unified_id")
router.resolve_model_name_from_model_id.assert_not_called()
router.get_model_ids.assert_not_called()
def _vertex_deployment(self):
deployment = MagicMock()
deployment.litellm_params.custom_llm_provider = "vertex_ai"
deployment.litellm_params.model = "vertex_ai/gemini-2.5-flash"
return deployment
def test_flag_on_routes_to_vertex_deployment(self):
"""Flag on: derive the bare model from the gs:// path, resolve it to a deployment id,
and use the raw unified_object_id as the provider batch id."""
router = MagicMock()
router.resolve_model_name_from_model_id.return_value = "gemini-2.5-flash"
router.get_model_ids.return_value = ["deploy-1"]
router.get_deployment = MagicMock(return_value=self._vertex_deployment())
instance = self._instance(track_unmanaged=True, router=router)
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(self._job(), MagicMock())
assert result == ("deploy-1", "8823717160934178816")
# bare model name (trailing GCS segment), not the full publishers/.. path
router.resolve_model_name_from_model_id.assert_called_once_with("gemini-2.5-flash")
router.get_model_ids.assert_called_once_with(model_name="gemini-2.5-flash")
def test_flag_on_routes_fine_tuned_endpoint_to_vertex_deployment(self):
"""A fine-tuned Gemini batch stores `endpoints/<id>` in the gs:// path; the bare model
(the endpoint id) must round-trip to the deployment configured as
`vertex_ai/gemini/<id>` (LIT-6899)."""
endpoint_id = "7768560373388541952"
router = MagicMock()
router.resolve_model_name_from_model_id.return_value = None
router.get_model_list.return_value = [
{
"model_name": "gemini-2.5-flash-dts-usc1",
"litellm_params": {
"model": f"vertex_ai/gemini/{endpoint_id}",
"custom_llm_provider": "vertex_ai",
},
"model_info": {"id": "deploy-ft"},
},
]
instance = self._instance(track_unmanaged=True, router=router)
job = self._job(
file_object=_unmanaged_vertex_file_object(
input_file_id=f"gs://bucket/litellm-vertex-files/endpoints/{endpoint_id}/abc.jsonl"
)
)
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(job, MagicMock())
assert result == ("deploy-ft", "8823717160934178816")
def test_flag_on_skips_non_vertex_deployment_sharing_model_group(self):
"""Flag on, but the only deployment for the model group is a non-vertex_ai
provider: must not be selected, even though the model group name matches."""
router = MagicMock()
router.resolve_model_name_from_model_id.return_value = "gemini-2.5-flash"
router.get_model_ids.return_value = ["deploy-openai"]
non_vertex_deployment = MagicMock()
non_vertex_deployment.litellm_params.custom_llm_provider = "openai"
non_vertex_deployment.litellm_params.model = "gpt-4o"
router.get_deployment = MagicMock(return_value=non_vertex_deployment)
instance = self._instance(track_unmanaged=True, router=router)
prom = MagicMock()
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(self._job(), prom)
assert result is None
prom.record_check_batch_cost_error.assert_called_once_with("unmanaged_no_matching_deployment")
def test_flag_on_uses_later_vertex_deployment_with_matching_suffix(self):
router = MagicMock()
router.resolve_model_name_from_model_id.return_value = "azure-gemini"
router.get_model_ids.return_value = ["deploy-azure"]
non_vertex_deployment = MagicMock()
non_vertex_deployment.litellm_params.custom_llm_provider = "azure"
non_vertex_deployment.litellm_params.model = "azure/gemini-2.5-flash"
router.get_deployment = MagicMock(return_value=non_vertex_deployment)
router.get_model_list.return_value = [
{
"model_name": "azure-gemini",
"litellm_params": {
"model": "azure/gemini-2.5-flash",
"custom_llm_provider": "azure",
},
"model_info": {"id": "deploy-azure"},
},
{
"model_name": "vertex-gemini",
"litellm_params": {
"model": "vertex_ai/gemini-2.5-flash",
"custom_llm_provider": "vertex_ai",
},
"model_info": {"id": "deploy-vertex"},
},
]
instance = self._instance(track_unmanaged=True, router=router)
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(self._job(), MagicMock())
assert result == ("deploy-vertex", "8823717160934178816")
router.get_model_ids.assert_called_once_with(model_name="azure-gemini")
def test_flag_on_no_matching_deployment_records_metric(self):
"""Flag on but no vertex_ai deployment for the model: skip with a distinct metric."""
router = MagicMock()
router.resolve_model_name_from_model_id.return_value = None
router.get_model_ids.return_value = []
instance = self._instance(track_unmanaged=True, router=router)
prom = MagicMock()
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(self._job(), prom)
assert result is None
prom.record_check_batch_cost_error.assert_called_once_with("unmanaged_no_matching_deployment")
def test_flag_on_non_gcs_input_is_not_unmanaged_vertex(self):
"""Flag on, but input_file_id is not a gs:// publishers path: treat as unroutable,
do not attempt model derivation."""
router = MagicMock()
instance = self._instance(track_unmanaged=True, router=router)
prom = MagicMock()
job = self._job(file_object=_unmanaged_vertex_file_object(input_file_id="file-abc-123"))
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(job, prom)
assert result is None
prom.record_check_batch_cost_error.assert_called_once_with("invalid_unified_id")
router.resolve_model_name_from_model_id.assert_not_called()
@pytest.mark.asyncio
async def test_end_to_end_costs_unmanaged_batch(self):
"""Flag on, completed unmanaged batch: the poller polls Vertex with the raw job id,
computes cost, and marks batch_processed=True. Fails before this change (the row is
skipped at the unified-id gate)."""
router = MagicMock()
router.resolve_model_name_from_model_id.return_value = "gemini-2.5-flash"
router.get_model_ids.return_value = ["deploy-1"]
mock_response: Final = LiteLLMBatch(
id="8823717160934178816",
status="completed",
output_file_id="gs://bucket/out/predictions.jsonl",
created_at=1,
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id="file-input-1",
object="batch",
)
router.aretrieve_batch = AsyncMock(return_value=mock_response)
router.get_deployment_credentials_with_provider = MagicMock(
return_value={"vertex_project": "p", "vertex_location": "us-central1"}
)
deployment = MagicMock()
deployment.litellm_params.custom_llm_provider = "vertex_ai"
deployment.litellm_params.model = "vertex_ai/gemini-2.5-flash"
deployment.model_name = "gemini-2.5-flash"
deployment.model_info.model_dump.return_value = {}
router.get_deployment = MagicMock(return_value=deployment)
instance = self._instance(track_unmanaged=True, router=router)
instance.proxy_logging_obj.get_proxy_hook.return_value = None
instance._has_batch_processed_column = True
prisma = instance.prisma_client
prisma.db = MagicMock()
prisma.db.litellm_managedobjecttable = MagicMock()
prisma.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
prisma.db.litellm_managedobjecttable.update = AsyncMock()
prisma.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[self._job()])
prisma.db.litellm_usertable = MagicMock()
prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
mock_file_content = MagicMock()
mock_file_content.content = b'{"id":"req-1"}'
with (
patch(_IS_B64, side_effect=[False, None]),
patch(
"litellm.files.main.afile_content",
new_callable=AsyncMock,
return_value=mock_file_content,
),
patch(
"litellm.batches.batch_utils.get_file_content_as_dictionary",
return_value=[{"id": "req-1"}],
),
patch(
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
new_callable=AsyncMock,
return_value=_batch_cost_result(
0.01,
{"prompt_tokens": 10, "completion_tokens": 5},
["gemini-2.5-flash"],
),
),
patch(
"litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
return_value=("gemini-2.5-flash", "vertex_ai", None, None),
),
patch("litellm.litellm_core_utils.litellm_logging.Logging") as mock_logging_cls,
):
mock_logging_obj = MagicMock()
mock_logging_obj.async_success_handler = AsyncMock()
mock_logging_cls.return_value = mock_logging_obj
await instance.check_batch_cost()
router.aretrieve_batch.assert_awaited_once()
assert router.aretrieve_batch.call_args[1]["model"] == "deploy-1"
assert router.aretrieve_batch.call_args[1]["batch_id"] == "8823717160934178816"
mock_logging_obj.async_success_handler.assert_awaited_once()
assert mock_logging_obj.async_success_handler.call_args[1]["batch_cost"] == 0.01
assert prisma.db.litellm_managedobjecttable.update_many.call_count == 2
update_data = prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
assert update_data["batch_processed"] is True
assert update_data["status"] == "complete"
class TestUnmanagedBedrockRouting:
"""Routing of unmanaged Bedrock batches whose unified_object_id is a raw model-invocation-job ARN."""
_ARN = "arn:aws:bedrock:us-east-1:298249409318:model-invocation-job/1ofb47x17jua"
def _instance(self, track_unmanaged, router):
from litellm_enterprise.proxy.common_utils.check_batch_cost import (
CheckBatchCost,
)
return CheckBatchCost(
proxy_logging_obj=MagicMock(),
prisma_client=MagicMock(),
llm_router=router,
track_unmanaged_batch_cost=track_unmanaged,
)
def _job(self, file_object=None):
job = MagicMock()
job.unified_object_id = self._ARN
job.file_object = file_object if file_object is not None else _unmanaged_bedrock_file_object()
return job
def _bedrock_deployment(self):
deployment = MagicMock()
deployment.litellm_params.custom_llm_provider = "bedrock"
deployment.litellm_params.model = "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
return deployment
def test_flag_off_skips_arn_unified_id_unchanged(self):
"""Default (flag off): a raw ARN unified_object_id is skipped exactly as before; no
model derivation or router lookup happens."""
router = MagicMock()
instance = self._instance(track_unmanaged=False, router=router)
prom = MagicMock()
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(self._job(), prom)
assert result is None
prom.record_check_batch_cost_error.assert_called_once_with("invalid_unified_id")
router.resolve_model_name_from_model_id.assert_not_called()
router.get_model_ids.assert_not_called()
def test_flag_on_routes_to_bedrock_deployment(self):
"""Flag on: derive the bare model from the s3:// object key (":" restored to "-" is
matched fuzzily), resolve it to a deployment id, and use the raw ARN as the batch id."""
router = MagicMock()
router.resolve_model_name_from_model_id.return_value = "claude-sonnet-4"
router.get_model_ids.return_value = ["deploy-1"]
router.get_deployment = MagicMock(return_value=self._bedrock_deployment())
instance = self._instance(track_unmanaged=True, router=router)
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(self._job(), MagicMock())
assert result == ("deploy-1", self._ARN)
def test_flag_on_skips_non_bedrock_deployment_sharing_model_group(self):
"""Flag on, but the only deployment for the model group is a non-bedrock provider:
must not be selected, even though the model group name matches."""
router = MagicMock()
router.resolve_model_name_from_model_id.return_value = "claude-sonnet-4"
router.get_model_ids.return_value = ["deploy-anthropic"]
non_bedrock_deployment = MagicMock()
non_bedrock_deployment.litellm_params.custom_llm_provider = "anthropic"
non_bedrock_deployment.litellm_params.model = "claude-sonnet-4-20250514"
router.get_deployment = MagicMock(return_value=non_bedrock_deployment)
instance = self._instance(track_unmanaged=True, router=router)
prom = MagicMock()
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(self._job(), prom)
assert result is None
prom.record_check_batch_cost_error.assert_called_once_with("unmanaged_no_matching_deployment")
def test_flag_on_matches_deployment_despite_colon_dash_mismatch(self):
"""The S3 object key has ':' replaced with '-' (e.g. 'v1-0'), but the configured
deployment's actual bedrock model id uses ':' (e.g. 'v1:0'). Routing must still match."""
router = MagicMock()
router.resolve_model_name_from_model_id.return_value = None
router.get_model_ids.return_value = []
router.get_model_list.return_value = [
{
"model_name": "claude-sonnet-4",
"litellm_params": {
"model": "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0",
"custom_llm_provider": "bedrock",
},
"model_info": {"id": "deploy-bedrock"},
}
]
instance = self._instance(track_unmanaged=True, router=router)
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(self._job(), MagicMock())
assert result == ("deploy-bedrock", self._ARN)
def test_flag_on_no_matching_deployment_records_metric(self):
"""Flag on but no bedrock deployment for the model: skip with a distinct metric."""
router = MagicMock()
router.resolve_model_name_from_model_id.return_value = None
router.get_model_ids.return_value = []
router.get_model_list.return_value = []
instance = self._instance(track_unmanaged=True, router=router)
prom = MagicMock()
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(self._job(), prom)
assert result is None
prom.record_check_batch_cost_error.assert_called_once_with("unmanaged_no_matching_deployment")
def test_flag_on_non_s3_input_is_not_unmanaged_bedrock(self):
"""Flag on, but input_file_id is not a litellm-bedrock-files- s3:// key: treat as
unroutable, do not attempt model derivation."""
router = MagicMock()
instance = self._instance(track_unmanaged=True, router=router)
prom = MagicMock()
job = self._job(file_object=_unmanaged_bedrock_file_object(input_file_id="file-abc-123"))
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(job, prom)
assert result is None
prom.record_check_batch_cost_error.assert_called_once_with("invalid_unified_id")
router.resolve_model_name_from_model_id.assert_not_called()
@pytest.mark.asyncio
async def test_end_to_end_costs_unmanaged_batch(self):
"""Flag on, completed unmanaged batch: the poller polls Bedrock with the raw ARN,
computes cost, and marks batch_processed=True."""
router = MagicMock()
router.resolve_model_name_from_model_id.return_value = "claude-sonnet-4"
router.get_model_ids.return_value = ["deploy-1"]
mock_response: Final = LiteLLMBatch(
id=self._ARN,
status="completed",
output_file_id="s3://bucket/out/predictions.jsonl",
created_at=1,
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id="file-input-1",
object="batch",
)
router.aretrieve_batch = AsyncMock(return_value=mock_response)
router.get_deployment_credentials_with_provider = MagicMock(return_value={"aws_region_name": "us-east-1"})
deployment = self._bedrock_deployment()
deployment.model_name = "claude-sonnet-4"
deployment.model_info.model_dump.return_value = {}
router.get_deployment = MagicMock(return_value=deployment)
instance = self._instance(track_unmanaged=True, router=router)
instance.proxy_logging_obj.get_proxy_hook.return_value = None
instance._has_batch_processed_column = True
prisma = instance.prisma_client
prisma.db = MagicMock()
prisma.db.litellm_managedobjecttable = MagicMock()
prisma.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
prisma.db.litellm_managedobjecttable.update = AsyncMock()
prisma.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[self._job()])
prisma.db.litellm_usertable = MagicMock()
prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
mock_file_content = MagicMock()
mock_file_content.content = b'{"id":"req-1"}'
with (
patch(_IS_B64, side_effect=[False, None]),
patch(
"litellm.files.main.afile_content",
new_callable=AsyncMock,
return_value=mock_file_content,
),
patch(
"litellm.batches.batch_utils.get_file_content_as_dictionary",
return_value=[{"id": "req-1"}],
),
patch(
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
new_callable=AsyncMock,
return_value=_batch_cost_result(
0.02,
{"prompt_tokens": 10, "completion_tokens": 5},
["claude-sonnet-4"],
),
),
patch(
"litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
return_value=("claude-sonnet-4", "bedrock", None, None),
),
patch("litellm.litellm_core_utils.litellm_logging.Logging") as mock_logging_cls,
):
mock_logging_obj = MagicMock()
mock_logging_obj.async_success_handler = AsyncMock()
mock_logging_cls.return_value = mock_logging_obj
await instance.check_batch_cost()
router.aretrieve_batch.assert_awaited_once()
assert router.aretrieve_batch.call_args[1]["model"] == "deploy-1"
assert router.aretrieve_batch.call_args[1]["batch_id"] == self._ARN
mock_logging_obj.async_success_handler.assert_awaited_once()
assert mock_logging_obj.async_success_handler.call_args[1]["batch_cost"] == 0.02
assert prisma.db.litellm_managedobjecttable.update_many.call_count == 2
update_data = prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
assert update_data["batch_processed"] is True
assert update_data["status"] == "complete"
class TestUnmanagedBatchCostFlagIsGeneralized:
"""The single track_unmanaged_batch_cost flag must cover both Vertex and Bedrock, not
just the provider it was originally added for."""
def test_one_flag_routes_both_vertex_and_bedrock_jobs(self):
from litellm_enterprise.proxy.common_utils.check_batch_cost import (
CheckBatchCost,
)
router = MagicMock()
router.resolve_model_name_from_model_id.side_effect = [
"gemini-2.5-flash",
"claude-sonnet-4",
]
router.get_model_ids.side_effect = [["deploy-vertex"], ["deploy-bedrock"]]
def _get_deployment(model_id):
if model_id == "deploy-vertex":
deployment = MagicMock()
deployment.litellm_params.custom_llm_provider = "vertex_ai"
deployment.litellm_params.model = "vertex_ai/gemini-2.5-flash"
return deployment
deployment = MagicMock()
deployment.litellm_params.custom_llm_provider = "bedrock"
deployment.litellm_params.model = "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
return deployment
router.get_deployment = MagicMock(side_effect=_get_deployment)
instance = CheckBatchCost(
proxy_logging_obj=MagicMock(),
prisma_client=MagicMock(),
llm_router=router,
track_unmanaged_batch_cost=True,
)
vertex_job = MagicMock()
vertex_job.unified_object_id = "8823717160934178816"
vertex_job.file_object = _unmanaged_vertex_file_object()
bedrock_job = MagicMock()
bedrock_job.unified_object_id = TestUnmanagedBedrockRouting._ARN
bedrock_job.file_object = _unmanaged_bedrock_file_object()
with patch(_IS_B64, return_value=False):
vertex_result = instance._resolve_job_routing(vertex_job, MagicMock())
bedrock_result = instance._resolve_job_routing(bedrock_job, MagicMock())
assert vertex_result == ("deploy-vertex", "8823717160934178816")
assert bedrock_result == ("deploy-bedrock", TestUnmanagedBedrockRouting._ARN)
class TestManagedOutputFileIdEncodesPublicModelGroup:
"""LIT-4964 regression: the unified output file id created by the background poller must
encode the public model group as ``target_model_names``, not the provider model.
Key model-access checks resolve a managed file id back to a model via ``target_model_names``,
so encoding the provider model (e.g. ``gpt-5.5``) makes
``GET /v1/files/{output_file_id}/content`` fail for every key.
"""
_PUBLIC_MODEL_GROUP = "gpt-5-batch"
_RAW_OUTPUT_FILE_ID = "file-batch-output-abc123"
@staticmethod
def _managed_input_file_id(model_group: str) -> str:
import base64
unified_id = (
"litellm_proxy:application/octet-stream;unified_id,c4843482-b176-4901-8292-7523fd0f2c6e;"
f"target_model_names,{model_group}"
)
return base64.urlsafe_b64encode(unified_id.encode()).decode().rstrip("=")
def _job(self, input_file_id: str) -> MagicMock:
from litellm.types.utils import LiteLLMBatch
job = MagicMock()
job.id = "job-lit-4964"
job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
job.created_by = "user-1"
job.team_id = None
job.file_object = LiteLLMBatch(
id="batch-456",
completion_window="24h",
created_at=1,
endpoint="/v1/chat/completions",
input_file_id=input_file_id,
object="batch",
status="completed",
).model_dump_json()
return job
async def _run(self, job: MagicMock) -> str:
from litellm_enterprise.proxy.common_utils.check_batch_cost import (
CheckBatchCost,
)
from enterprise.litellm_enterprise.proxy.hooks.managed_files import PROXY_LiteLLMManagedFiles
from litellm.proxy.openai_files_endpoints.common_utils import ManagedBatchOutputFileWriter
from litellm.types.utils import LiteLLMBatch
router = MagicMock()
router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})
deployment = MagicMock()
deployment.litellm_params.custom_llm_provider = "azure"
deployment.litellm_params.model = "azure/gpt-5.5"
deployment.model_name = self._PUBLIC_MODEL_GROUP
deployment.model_info.model_dump.return_value = {}
router.get_deployment = MagicMock(return_value=deployment)
hook = MagicMock(spec=ManagedBatchOutputFileWriter)
hook.get_unified_output_file_id = lambda output_file_id, model_id, model_name: (
PROXY_LiteLLMManagedFiles.get_unified_output_file_id(
None, output_file_id=output_file_id, model_id=model_id, model_name=model_name
)
)
hook.store_batch_output_file = AsyncMock()
proxy_logging_obj = MagicMock()
proxy_logging_obj.get_proxy_hook.return_value = hook
prisma_client = MagicMock()
prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
instance = CheckBatchCost(
proxy_logging_obj=proxy_logging_obj,
prisma_client=prisma_client,
llm_router=router,
)
response = LiteLLMBatch(
id="batch-456",
completion_window="24h",
created_at=1,
endpoint="/v1/chat/completions",
input_file_id=job.file_object,
object="batch",
status="completed",
)
response.output_file_id = self._RAW_OUTPUT_FILE_ID
file_content = MagicMock()
file_content.content = b'{"id":"req-1"}'
with (
patch(
"litellm.files.main.afile_content",
new_callable=AsyncMock,
return_value=file_content,
),
patch(
"litellm.batches.batch_utils.get_file_content_as_dictionary",
return_value=[{"id": "req-1"}],
),
patch(
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
new_callable=AsyncMock,
return_value=_batch_cost_result(0.01, {"prompt_tokens": 10}, ["gpt-5.5"]),
),
patch("litellm.litellm_core_utils.litellm_logging.Logging") as logging_cls,
):
logging_obj = MagicMock()
logging_obj.async_success_handler = AsyncMock()
logging_cls.return_value = logging_obj
await instance._track_completed_batch_cost(
job=job,
response=response,
model_id="model-123",
batch_id="batch-456",
prom_logger=None,
)
return response.output_file_id
@pytest.mark.asyncio
async def test_target_model_names_comes_from_input_file_not_provider_model(self):
from litellm.proxy.openai_files_endpoints.common_utils import (
is_base64_encoded_unified_file_id,
get_models_from_unified_file_id,
)
output_file_id = await self._run(self._job(self._managed_input_file_id(self._PUBLIC_MODEL_GROUP)))
decoded = is_base64_encoded_unified_file_id(output_file_id)
assert get_models_from_unified_file_id(decoded) == [self._PUBLIC_MODEL_GROUP]
assert "gpt-5.5" not in decoded
@pytest.mark.asyncio
async def test_key_scoped_to_model_group_can_read_the_output_file(self):
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.auth_checks import can_key_call_model
from litellm.proxy.auth.auth_utils import (
_extract_models_from_managed_resource_id,
)
output_file_id = await self._run(self._job(self._managed_input_file_id(self._PUBLIC_MODEL_GROUP)))
models = _extract_models_from_managed_resource_id(output_file_id, "file_id", None)
assert models == [self._PUBLIC_MODEL_GROUP]
assert (
await can_key_call_model(
model=models[0],
llm_model_list=None,
valid_token=UserAPIKeyAuth(api_key="sk-test", models=[self._PUBLIC_MODEL_GROUP]),
llm_router=None,
)
is True
)
@pytest.mark.asyncio
async def test_falls_back_to_deployment_model_group_without_managed_input_file(self):
from litellm.proxy.openai_files_endpoints.common_utils import (
is_base64_encoded_unified_file_id,
get_models_from_unified_file_id,
)
output_file_id = await self._run(self._job("file-raw-provider-input"))
decoded = is_base64_encoded_unified_file_id(output_file_id)
assert get_models_from_unified_file_id(decoded) == [self._PUBLIC_MODEL_GROUP]
class TestBatchCostAttribution:
"""CheckBatchCost rebuilds the creator's spend metadata from the managed-object row so
the batch-cost log is attributed like a non-batch request."""
def _prisma(self, key_row=None, team_row=None, user_row=None):
prisma = MagicMock()
prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=key_row)
prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=team_row)
prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=user_row)
return prisma
def _instance(self, key_row=None, team_row=None, user_row=None, prisma=None):
from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
ManagedObjectCreatorAttribution,
)
return ManagedObjectCreatorAttribution(
prisma_client=prisma if prisma is not None else self._prisma(key_row, team_row, user_row),
poller_name="CheckBatchCost",
)
def _job(self, **overrides):
from types import SimpleNamespace
fields = {
"created_by": "alice",
"team_id": "team-alpha",
"api_key": "hash-alice",
"request_tags": ["env:prod"],
}
fields.update(overrides)
return SimpleNamespace(unified_object_id="uoi", **fields)
@pytest.mark.asyncio
async def test_metadata_carries_key_team_and_tags(self):
"""The spend row names the creating key, its team, both aliases, and the tags."""
from types import SimpleNamespace
instance = self._instance(
key_row=SimpleNamespace(key_alias="prod-key"),
team_row=SimpleNamespace(team_alias="Team Alpha"),
user_row=SimpleNamespace(user_email="[email protected]", user_alias=None),
)
metadata = await instance.batch_metadata(self._job(), "batch-1")
assert metadata["user_api_key"] == "hash-alice"
assert metadata["user_api_key_hash"] == "hash-alice"
assert metadata["user_api_key_user_id"] == "alice"
assert metadata["user_api_key_team_id"] == "team-alpha"
assert metadata["user_api_key_alias"] == "prod-key"
assert metadata["user_api_key_team_alias"] == "Team Alpha"
assert metadata["tags"] == ["env:prod"]
@pytest.mark.asyncio
async def test_metadata_tolerates_legacy_row_without_columns(self):
"""Rows created before the columns existed carry only created_by/team_id and must
still produce an attributed row rather than raising."""
instance = self._instance()
job = self._job(api_key=None, request_tags=None)
metadata = await instance.batch_metadata(job, "batch-1")
assert metadata["user_api_key"] is None
assert metadata["user_api_key_user_id"] == "alice"
assert metadata["user_api_key_team_id"] == "team-alpha"
assert "tags" not in metadata
@pytest.mark.asyncio
async def test_metadata_keeps_key_when_team_key_has_no_user(self):
"""A team-scoped key carries no user id. The user lookup is skipped (prisma rejects
a None user_id) and the key hash still drives key-level attribution."""
from types import SimpleNamespace
prisma = self._prisma(key_row=SimpleNamespace(key_alias="svc-key"))
instance = self._instance(prisma=prisma)
job = self._job(created_by=None)
metadata = await instance.batch_metadata(job, "batch-1")
assert metadata["user_api_key"] == "hash-alice"
assert metadata["user_api_key_user_id"] is None
assert metadata["user_api_key_alias"] == "svc-key"
prisma.db.litellm_usertable.find_unique.assert_not_called()
@pytest.mark.asyncio
async def test_batch_metadata_keeps_the_creator_column_and_adds_no_budgets(self):
"""The batch cost event keeps the user id and fields the batch poll has always written,
even though background responses read budgets and drop key-scoped creators off the same rows."""
from types import SimpleNamespace
instance = self._instance(
key_row=SimpleNamespace(key_alias="svc-key", spend=4.0, max_budget=5.0),
team_row=SimpleNamespace(team_alias="Team Alpha", organization_id=None, spend=40.0, max_budget=50.0),
)
metadata = await instance.batch_metadata(self._job(created_by="key:hash-alice"), "batch-1")
assert metadata["user_api_key_user_id"] == "key:hash-alice"
assert [field for field in metadata if field.endswith(("_spend", "_max_budget"))] == []
@pytest.mark.asyncio
async def test_metadata_drops_non_string_tags(self):
"""Non-string tags are dropped so a malformed stored value cannot slip past the
tag-budget checks that consume this metadata."""
instance = self._instance()
job = self._job(request_tags=["env:prod", 7, None, "team:ml"])
metadata = await instance.batch_metadata(job, "batch-1")
assert metadata["tags"] == ["env:prod", "team:ml"]
@pytest.mark.asyncio
async def test_key_alias_lookup_failure_does_not_break_attribution(self):
"""An alias lookup failure must not lose the spend row; the key hash and team still
attribute it."""
prisma = self._prisma()
prisma.db.litellm_verificationtoken.find_unique = AsyncMock(side_effect=Exception("db down"))
instance = self._instance(prisma=prisma)
metadata = await instance.batch_metadata(self._job(), "batch-1")
assert metadata["user_api_key"] == "hash-alice"
assert metadata.get("user_api_key_alias") is None
@pytest.mark.asyncio
async def test_cli_session_batch_keeps_its_alias_without_a_key_row(self):
instance = self._instance(key_row=None)
metadata = await instance.batch_metadata(self._job(api_key="cli-session-alice"), "batch-1")
assert metadata["user_api_key"] == "cli-session-alice"
assert metadata["user_api_key_alias"] == "cli-session-alice"
@pytest.mark.asyncio
async def test_raw_cli_session_token_on_a_legacy_batch_row_is_not_treated_as_the_alias(self):
instance = self._instance(key_row=None)
metadata = await instance.batch_metadata(self._job(api_key="cli-session-Qm7xJ2kP9sLw4vT1nR8yAa"), "batch-1")
assert metadata.get("user_api_key_alias") is None
@pytest.mark.asyncio
async def test_unnamed_key_keeps_the_creating_user_alias(self):
"""Regression: a key generated without key_alias resolves to no alias, and the
overwrite must not null out the creating user's alias that _get_user_info supplied.
Most keys carry no alias, so this is the common batch, not an edge case."""
from types import SimpleNamespace
instance = self._instance(
key_row=SimpleNamespace(key_alias=None),
user_row=SimpleNamespace(user_email="[email protected]", user_alias="Alice Chen"),
)
metadata = await instance.batch_metadata(self._job(), "batch-1")
assert metadata["user_api_key_alias"] == "Alice Chen"
assert metadata["user_api_key"] == "hash-alice"
@pytest.mark.asyncio
async def test_rotated_key_keeps_the_creating_user_alias(self):
"""Batches outlive keys. When the creating key has been rotated or deleted the
lookup returns no row, and the spend log keeps a resolvable name instead of null."""
from types import SimpleNamespace
instance = self._instance(
key_row=None,
user_row=SimpleNamespace(user_email="[email protected]", user_alias="Alice Chen"),
)
metadata = await instance.batch_metadata(self._job(), "batch-1")
assert metadata["user_api_key_alias"] == "Alice Chen"
@pytest.mark.asyncio
async def test_named_key_still_owns_the_alias(self):
"""The fallback must not weaken the intended precedence: a key that has its own
alias still overrides the creating user's."""
from types import SimpleNamespace
instance = self._instance(
key_row=SimpleNamespace(key_alias="prod-key"),
user_row=SimpleNamespace(user_email="[email protected]", user_alias="Alice Chen"),
)
metadata = await instance.batch_metadata(self._job(), "batch-1")
assert metadata["user_api_key_alias"] == "prod-key"
@pytest.mark.asyncio
async def test_org_id_snapshotted_on_the_row_wins(self):
"""The org_id column captures the creating key's organization at submission time,
like team_id, so a key later moved to another org still bills the original one."""
from types import SimpleNamespace
instance = self._instance(
key_row=SimpleNamespace(key_alias="prod-key", organization_id="org-moved-to"),
team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"),
)
metadata = await instance.batch_metadata(self._job(org_id="org-at-creation"), "batch-1")
assert metadata["user_api_key_org_id"] == "org-at-creation"
@pytest.mark.asyncio
async def test_org_id_comes_from_the_creating_key(self):
"""The spend update writer increments organization spend from user_api_key_org_id.
A legacy row without the org_id column falls back to the creating key's org."""
from types import SimpleNamespace
instance = self._instance(
key_row=SimpleNamespace(key_alias="prod-key", organization_id="org-42"),
team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"),
)
metadata = await instance.batch_metadata(self._job(), "batch-1")
assert metadata["user_api_key_org_id"] == "org-42"
@pytest.mark.asyncio
async def test_org_id_falls_back_to_the_team_organization(self):
"""A key with no org of its own still books batch spend against its team's
organization, matching how the request path resolves org attribution."""
from types import SimpleNamespace
instance = self._instance(
key_row=SimpleNamespace(key_alias="prod-key", organization_id=None),
team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"),
)
metadata = await instance.batch_metadata(self._job(), "batch-1")
assert metadata["user_api_key_org_id"] == "org-team"
@pytest.mark.asyncio
async def test_key_lookup_failure_still_bills_the_team_org(self):
"""A key-table error while resolving a legacy row's org must not drop the team's
organization: the two lookups fail independently, so org spend still lands."""
from types import SimpleNamespace
prisma = self._prisma(team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"))
prisma.db.litellm_verificationtoken.find_unique = AsyncMock(side_effect=Exception("db down"))
instance = self._instance(prisma=prisma)
metadata = await instance.batch_metadata(self._job(), "batch-1")
assert metadata["user_api_key_org_id"] == "org-team"
@pytest.mark.asyncio
async def test_no_org_leaves_the_key_unset(self):
"""Without any org the key is absent entirely, so the spend writer's org update
stays skipped instead of matching an empty-string organization."""
from types import SimpleNamespace
instance = self._instance(
key_row=SimpleNamespace(key_alias="prod-key", organization_id=None),
team_row=SimpleNamespace(team_alias="Team Alpha", organization_id=None),
)
metadata = await instance.batch_metadata(self._job(), "batch-1")
assert "user_api_key_org_id" not in metadata
@pytest.mark.asyncio
async def test_metadata_provenance_keeps_spend_log_api_key_joinable(self):
"""
CheckBatchCost stores the VerificationToken hash on the managed object. The
spend-log writer must receive matching user_api_key_hash provenance so it
does not re-hash that value; otherwise DailyUserSpend.api_key no longer joins
VerificationToken and Usage shows key-hash-... with a null alias/email.
"""
from datetime import datetime, timezone
from types import SimpleNamespace
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
from litellm.proxy.utils import hash_token
token_hash = hash_token("sk-batch-creator-key")
instance = self._instance(
key_row=SimpleNamespace(key_alias="prod-key"),
user_row=SimpleNamespace(user_email="[email protected]", user_alias=None),
)
metadata = await instance.batch_metadata(self._job(api_key=token_hash), "batch-1")
assert metadata["user_api_key"] == token_hash
assert metadata["user_api_key_hash"] == token_hash
payload = get_logging_payload(
kwargs={
"model": "gpt-4o",
"call_type": "aretrieve_batch",
"litellm_params": {"metadata": metadata},
},
response_obj={
"id": "batch_123",
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
},
start_time=datetime.now(timezone.utc),
end_time=datetime.now(timezone.utc),
)
assert payload["api_key"] == token_hash
assert payload["api_key"] != hash_token(token_hash)
class TestPollPageStarvation:
"""LIT-5462 regression: a row that can never be costed used to keep its slot in the
MAX_OBJECTS_PER_POLL_CYCLE page forever, so once enough of them accumulated no newer
batch was ever polled or costed."""
def _instance(self, prisma, llm_router):
from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost
proxy_logging_obj = MagicMock()
proxy_logging_obj.get_proxy_hook.return_value = None
return CheckBatchCost(
proxy_logging_obj=proxy_logging_obj,
prisma_client=prisma,
llm_router=llm_router,
)
def _prisma(self, jobs):
prisma = MagicMock()
prisma.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=0)
prisma.db.litellm_managedobjecttable.update = AsyncMock()
prisma.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=jobs)
prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
return prisma
def _job(self, job_id, unified_object_id):
job = MagicMock()
job.id = job_id
job.unified_object_id = unified_object_id
job.created_by = "user-1"
return job
@staticmethod
def _encode(unified_id: str) -> str:
import base64
return base64.urlsafe_b64encode(unified_id.encode()).decode().rstrip("=")
@pytest.mark.asyncio
async def test_unified_id_without_model_id_is_retired(self):
"""A unified id that decodes but carries no model_id is unroutable no matter what
the config says, so it must leave the poll page instead of being retried forever."""
prisma = self._prisma([self._job("job-no-model", self._encode("litellm_proxy;llm_batch_id:poison-no-model"))])
llm_router = MagicMock()
llm_router.aretrieve_batch = AsyncMock()
await self._instance(prisma, llm_router).check_batch_cost()
llm_router.aretrieve_batch.assert_not_awaited()
prisma.db.litellm_managedobjecttable.update_many.assert_awaited()
call = prisma.db.litellm_managedobjecttable.update_many.call_args[1]
assert call["where"]["id"] == "job-no-model"
assert call["where"]["batch_processed"] is False
assert call["data"] == {"batch_processed": True}
@pytest.mark.asyncio
async def test_provider_404_retires_job(self):
"""The provider dropping its record of the batch is permanent: no later retrieve
can succeed, so the row must stop occupying a slot."""
import litellm
prisma = self._prisma(
[
self._job(
"job-gone",
self._encode("litellm_proxy;model_id:model-123;llm_batch_id:batch_deadbeef"),
)
]
)
llm_router = MagicMock()
llm_router.aretrieve_batch = AsyncMock(
side_effect=litellm.NotFoundError(
message="No batch found with id 'batch_deadbeef'.",
model="model-123",
llm_provider="openai",
)
)
await self._instance(prisma, llm_router).check_batch_cost()
prisma.db.litellm_managedobjecttable.update_many.assert_awaited()
assert prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"] == {"batch_processed": True}
@pytest.mark.asyncio
async def test_provider_404_with_deployment_gone_keeps_job(self):
"""With the batch's own deployment removed from the router, default fallbacks can
send the retrieve to a provider that never saw the batch. That 404 proves nothing,
so the row must stay unprocessed instead of losing its spend forever."""
import litellm
prisma = self._prisma(
[
self._job(
"job-misrouted",
self._encode("litellm_proxy;model_id:model-gone;llm_batch_id:batch_alive"),
)
]
)
llm_router = MagicMock()
llm_router.get_deployment = MagicMock(return_value=None)
llm_router.aretrieve_batch = AsyncMock(
side_effect=litellm.NotFoundError(
message="No batch found with id 'batch_alive'.",
model="model-gone",
llm_provider="openai",
)
)
await self._instance(prisma, llm_router).check_batch_cost()
prisma.db.litellm_managedobjecttable.update.assert_not_awaited()
@pytest.mark.asyncio
async def test_transient_provider_error_keeps_job_for_retry(self):
"""A failure that may clear up (timeout, 5xx) must still leave the row unprocessed."""
prisma = self._prisma(
[
self._job(
"job-flaky",
self._encode("litellm_proxy;model_id:model-123;llm_batch_id:batch_flaky"),
)
]
)
llm_router = MagicMock()
llm_router.aretrieve_batch = AsyncMock(side_effect=Exception("connection reset"))
await self._instance(prisma, llm_router).check_batch_cost()
prisma.db.litellm_managedobjecttable.update.assert_not_awaited()
@pytest.mark.asyncio
async def test_retirement_falls_back_to_status_without_batch_processed_column(self):
"""Older schemas have no batch_processed column, so the only way to stop selecting
the row is the status filter the poll query already applies."""
prisma = self._prisma([self._job("job-legacy", self._encode("litellm_proxy;llm_batch_id:poison-no-model"))])
instance = self._instance(prisma, MagicMock())
instance._has_batch_processed_column = False
await instance.check_batch_cost()
assert prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"] == {"status": "stale_expired"}
@pytest.mark.asyncio
async def test_newer_batch_is_polled_once_dead_rows_are_retired(self):
"""The end state the customer cares about: dead rows retire on the cycle they are
first seen, and the healthy batch behind them keeps getting polled."""
dead_rows = [
self._job("job-no-model", self._encode("litellm_proxy;llm_batch_id:poison-no-model")),
self._job(
"job-gone",
self._encode("litellm_proxy;model_id:model-123;llm_batch_id:batch_deadbeef"),
),
]
live_row = self._job(
"job-live",
self._encode("litellm_proxy;model_id:model-123;llm_batch_id:batch_live"),
)
prisma = self._prisma(dead_rows + [live_row])
import litellm
in_progress = MagicMock()
in_progress.status = "in_progress"
async def _retrieve(model, batch_id, litellm_metadata):
if batch_id == "batch_deadbeef":
raise litellm.NotFoundError(
message=f"No batch found with id '{batch_id}'.",
model=model,
llm_provider="openai",
)
return in_progress
llm_router = MagicMock()
llm_router.aretrieve_batch = AsyncMock(side_effect=_retrieve)
await self._instance(prisma, llm_router).check_batch_cost()
retired = [
call[1]["where"]["id"]
for call in prisma.db.litellm_managedobjecttable.update_many.call_args_list
if "id" in call[1]["where"]
]
assert retired == ["job-no-model", "job-gone"]
assert llm_router.aretrieve_batch.await_args_list[-1][1]["batch_id"] == "batch_live", (
"the newer healthy batch must still be polled in the same cycle"
)
@pytest.mark.asyncio
async def test_404_that_does_not_name_the_batch_keeps_job_for_retry(self):
"""A 404 about something other than the batch, e.g. a renamed Azure deployment, is
fixable in config, so the row must survive to be costed after the fix."""
import litellm
prisma = self._prisma(
[
self._job(
"job-bad-deployment",
self._encode("litellm_proxy;model_id:model-123;llm_batch_id:batch_real"),
)
]
)
llm_router = MagicMock()
llm_router.aretrieve_batch = AsyncMock(
side_effect=litellm.NotFoundError(
message="Error code: 404 - DeploymentNotFound",
model="model-123",
llm_provider="azure",
)
)
await self._instance(prisma, llm_router).check_batch_cost()
prisma.db.litellm_managedobjecttable.update.assert_not_awaited()
class _FakeManagedObjectRow:
"""One managed batch row the provider has finished but nothing has costed yet."""
def __init__(self):
self.id = "job-claim-1"
self.unified_object_id = _CLAIM_UNIFIED_BATCH_ID
self.model_object_id = "batch-456"
self.file_purpose = "batch"
self.status = "in_progress"
self.batch_processed = False
self.created_by = "user-1"
self.team_id = None
self.api_key = None
self.request_tags = None
self.created_at = datetime(9999, 1, 1, tzinfo=timezone.utc)
self.updated_at = datetime(2000, 1, 1, tzinfo=timezone.utc)
self.file_object = json.dumps(
{
"id": "batch-456",
"status": "in_progress",
"input_file_id": "file-input-1",
"output_file_id": _CLAIM_OUTPUT_FILE_ID,
}
)
class _FakeManagedObjectTable:
"""A LiteLLM_ManagedObjectTable double backed by one real, mutable row.
It honours the batch_processed and status filters, so the poller's compare-and-swap
and the managed-files deletion guard both read the same state a shared Postgres row
would give them. Staleness sweeps (the only queries scoped by created_at) never match.
"""
def __init__(self, row: _FakeManagedObjectRow, journal: list):
self.row = row
self.journal = journal
self.update_many = AsyncMock(side_effect=self._update_many)
self.update = AsyncMock(side_effect=self._update)
self.find_many = AsyncMock(side_effect=self._find_many)
self.find_first = AsyncMock(return_value=None)
def _matches(self, where: dict) -> bool:
for key, value in where.items():
if key == "created_at":
if "lt" in value and not self.row.created_at < value["lt"]:
return False
continue
if key == "status":
if self.row.status in value.get("not_in", []):
return False
if "in" in value and self.row.status not in value["in"]:
return False
elif getattr(self.row, key) != value:
return False
return True
async def _update_many(self, *, where: dict, data: dict) -> int:
if not self._matches(where):
return 0
if "updated_at" in where:
self.journal.append("finalize")
elif "batch_processed" in where:
self.journal.append("claim" if data.get("batch_processed") else "release")
for key, value in data.items():
setattr(self.row, key, value)
return 1
async def _update(self, *, where: dict, data: dict) -> None:
self.journal.append("finalize")
for key, value in data.items():
setattr(self.row, key, value)
async def _find_many(self, *, where: dict, take=None, order=None) -> list:
return [self.row] if self._matches(where) else []
class TestMultiPodBatchCostClaim:
"""LIT-4827 regression: every pod and uvicorn worker schedules its own poller against
the shared LiteLLM_ManagedObjectTable, so a completed batch must be claimed atomically
before its cost is logged. Without the claim two pods select the same row in one window
and both write an aretrieve_batch spend log for it, double counting the spend.
The claim sits immediately before the spend-log write rather than before the results
fetch, because batch_processed is also what keeps an unbilled row selectable by later
poll cycles and what blocks deletion of the files the fetch reads."""
@staticmethod
def _instance(prisma, llm_router):
from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost
proxy_logging_obj = MagicMock()
proxy_logging_obj.get_proxy_hook.return_value = None
return CheckBatchCost(
proxy_logging_obj=proxy_logging_obj,
prisma_client=prisma,
llm_router=llm_router,
)
@staticmethod
def _prisma(row: _FakeManagedObjectRow, journal: list):
prisma = MagicMock()
prisma.db.litellm_managedobjecttable = _FakeManagedObjectTable(row, journal)
prisma.db.litellm_managedfiletable.find_many = AsyncMock(return_value=[])
prisma.db.litellm_managedfiletable.find_first = AsyncMock(return_value=None)
prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None)
return prisma
@staticmethod
def _router():
from litellm.types.utils import LiteLLMBatch
response: Final = LiteLLMBatch(
id="batch-456",
status="completed",
output_file_id=_CLAIM_OUTPUT_FILE_ID,
error_file_id=None,
created_at=1,
completed_at=2,
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id="file-input-1",
object="batch",
)
deployment = MagicMock()
deployment.litellm_params.custom_llm_provider = "openai"
deployment.litellm_params.model = "gpt-4"
deployment.model_info.model_dump.return_value = {}
router = MagicMock()
router.aretrieve_batch = AsyncMock(return_value=response)
router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})
router.get_deployment = MagicMock(return_value=deployment)
return router
@staticmethod
@contextmanager
def _billing_patches(journal: list, during_fetch=None, bill_error=None):
"""Patch the cost path a batch runs through, journalling the results fetch and the
spend-log write. during_fetch runs while the output file is being read, which is
the window an interrupted worker or a concurrent file deletion lands in."""
file_content = MagicMock()
file_content.content = b'{"id":"req-1"}'
async def _afile_content(**kwargs):
journal.append("fetch")
if during_fetch is not None:
await during_fetch()
return file_content
async def _bill(**kwargs):
journal.append("bill")
if bill_error is not None:
raise bill_error
def _is_b64(file_id):
if file_id == _CLAIM_UNIFIED_BATCH_ID:
return "llm_model_id,model-123;llm_batch_id,batch-456;"
return False
logging_obj = MagicMock()
logging_obj.async_success_handler = AsyncMock(side_effect=_bill)
with (
patch(_IS_B64, side_effect=_is_b64),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
return_value="model-123",
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
return_value="batch-456",
),
patch("litellm.files.main.afile_content", new=AsyncMock(side_effect=_afile_content)),
patch(
"litellm.batches.batch_utils.get_file_content_as_dictionary",
return_value=[{"id": "req-1"}],
),
patch(
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
new_callable=AsyncMock,
return_value=_batch_cost_result(0.01, {"prompt_tokens": 10, "completion_tokens": 5}, ["gpt-4"]),
),
patch(
"litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
return_value=("gpt-4", "openai", None, None),
),
patch("litellm.litellm_core_utils.litellm_logging.Logging", return_value=logging_obj),
):
yield logging_obj
@pytest.mark.asyncio
@pytest.mark.parametrize("failure_phase", ("before_write", "after_write", "external"))
@pytest.mark.parametrize("collector_enabled", (False, True))
async def test_builtin_accounting_retries_only_before_a_possible_charge(
self, failure_phase: str, collector_enabled: bool
) -> None:
import base64
from datetime import datetime
import httpx
import litellm
import respx
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
from litellm.proxy.utils import PrismaClient
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
from litellm.types.utils import SpecialEnums
class AccountingDependency:
def __init__(self) -> None:
self.failure_phase = failure_phase
self.writer: Final = DBSpendUpdateWriter()
self.failed_tracking_alert = AsyncMock()
self.slack_alerting_instance = MagicMock()
self.slack_alerting_instance.customer_spend_alert = AsyncMock(
side_effect=ConnectionError("Alert destination unavailable after charge")
if failure_phase == "after_write"
else None
)
@property
def db_spend_update_writer(self) -> DBSpendUpdateWriter:
if self.failure_phase == "before_write":
raise ConnectionError("Accounting dependency temporarily unavailable")
return self.writer
class ExternalCallback(CustomLogger):
async def async_log_success_event(
self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
) -> None:
if failure_phase == "external":
raise ConnectionError("External destination unavailable")
budget_cache: Final = DualCache()
RouterBudgetLimiting(
dual_cache=budget_cache,
provider_budget_config={"openai": {"budget_limit": 100, "time_period": "1d"}},
model_list=[
{
"model_name": "batch-alias",
"litellm_params": {"model": "openai/batch-test-model", "max_budget": 100, "budget_duration": "1d"},
"model_info": {"id": "model-123"},
}
],
)
dependency: Final = AccountingDependency()
spend_prisma: Final = PrismaClient(
database_url="postgresql://unused:unused@localhost:5432/unused", proxy_logging_obj=dependency
)
create_spend_rows: Final = AsyncMock(return_value=1)
input_rate: Final = 0.000002
litellm.register_model(
{
"batch-test-model": {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token_batches": input_rate,
"output_cost_per_token_batches": 0.000006,
}
}
)
content: Final = json.dumps(
{
"response": {
"status_code": 200,
"body": {
"model": "batch-test-model",
"usage": {"prompt_tokens": 100, "completion_tokens": 0, "total_tokens": 100},
},
},
"error": None,
}
).encode()
row: Final = _FakeManagedObjectRow()
original_status: Final = row.status
original_file_object: Final = row.file_object
row.unified_object_id = (
base64.urlsafe_b64encode(
SpecialEnums.LITELLM_MANAGED_BATCH_COMPLETE_STR.value.format("model-123", "batch-456").encode()
)
.decode()
.rstrip("=")
)
prisma: Final = self._prisma(row, [])
router: Final = self._router()
router.get_deployment.return_value = Deployment(
model_name="batch-alias",
litellm_params=LiteLLM_Params(model="openai/batch-test-model"),
model_info=ModelInfo(id="model-123"),
)
from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer
producer: Final = MagicMock(spec=SpendEventProducer) if collector_enabled else None
litellm.logging_callback_manager.add_litellm_async_success_callback(
ProxyDBLogger(spend_event_producer=producer)
)
litellm.logging_callback_manager.add_litellm_async_success_callback(ExternalCallback())
with (
respx.mock,
patch("litellm.proxy.proxy_server.proxy_logging_obj", dependency),
patch("litellm.proxy.proxy_server.prisma_client", spend_prisma),
patch.object(
spend_prisma.writer_db._original_prisma,
"litellm_spendlogs",
MagicMock(create_many=create_spend_rows),
),
):
respx.get(f"https://api.openai.com/v1/files/{_CLAIM_OUTPUT_FILE_ID}/content").mock(
return_value=httpx.Response(200, content=content)
)
worker: Final = self._instance(prisma, router)
await worker.check_batch_cost()
assert row.batch_processed is (failure_phase != "before_write")
if failure_phase == "before_write":
assert row.status == original_status
assert json.loads(row.file_object) == original_file_object
assert create_spend_rows.await_count == (0 if failure_phase == "before_write" else 1)
dependency.failure_phase = "recovered"
await worker.check_batch_cost()
await worker.check_batch_cost()
assert row.batch_processed is True
assert await budget_cache.async_get_cache("provider_spend:openai:1d") == pytest.approx(100 * input_rate)
assert await budget_cache.async_get_cache("deployment_spend:model-123:1d") == pytest.approx(100 * input_rate)
create_spend_rows.assert_awaited_once()
assert create_spend_rows.call_args.kwargs["data"][0]["spend"] == pytest.approx(100 * input_rate)
if producer is not None:
assert all(
json.loads(call.args[0])["call_type"] != "aretrieve_batch" for call in producer.publish.call_args_list
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_model", "successes", "failures", "expected_model"),
(
("openai/", 0, 2, "openai/"),
("", 0, 2, "model-123"),
("openai/", 2, 1, "batch-test-model"),
("openai/batch-test-model", 2, 1, "batch-test-model"),
("openai/batch-test-model", 2, 0, "batch-test-model"),
),
)
@pytest.mark.parametrize("failure_envelope", (None, 1, "invalid", [1]))
async def test_batch_accounting_preserves_models_and_mixed_results(
self, deployment_model: str, successes: int, failures: int, expected_model: str, failure_envelope: object
) -> None:
import base64
from datetime import datetime
import httpx
import litellm
import respx
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
from litellm.types.utils import LiteLLMBatch, SpecialEnums
input_rate: Final = 0.000002
output_rate: Final = 0.000006
litellm.register_model(
{
"batch-test-model": {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token_batches": input_rate,
"output_cost_per_token_batches": output_rate,
}
}
)
success: Final = {
"response": {
"status_code": 200,
"body": {
"model": "batch-test-model",
"usage": {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120},
},
},
"error": None,
}
failure: Final = {"response": failure_envelope, "error": {"code": "batch_expired", "message": "expired"}}
rows: Final = (success,) * successes + (failure,) * failures
content: Final = b"\n".join(json.dumps(row).encode() for row in rows)
class BatchSpendRecorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.calls: tuple[dict[str, object], ...] = ()
async def async_log_success_event(
self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
) -> None:
if kwargs["call_type"] == "aretrieve_batch":
self.calls += (kwargs,)
callback: Final = BatchSpendRecorder()
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
row: Final = _FakeManagedObjectRow()
row.unified_object_id = (
base64.urlsafe_b64encode(
SpecialEnums.LITELLM_MANAGED_BATCH_COMPLETE_STR.value.format("model-123", "batch-456").encode()
)
.decode()
.rstrip("=")
)
prisma: Final = self._prisma(row, [])
router: Final = self._router()
router.get_deployment.return_value = Deployment(
model_name="batch-alias",
litellm_params=LiteLLM_Params(model=deployment_model, custom_llm_provider="openai"),
model_info=ModelInfo(id="model-123"),
)
router.aretrieve_batch.return_value = LiteLLMBatch(
id="batch-456",
completion_window="24h",
created_at=1,
endpoint="/v1/chat/completions",
input_file_id="file-input-1",
output_file_id=_CLAIM_OUTPUT_FILE_ID,
object="batch",
status="completed",
)
router.get_deployment_credentials_with_provider.return_value = {
"api_key": "synthetic-key",
"custom_llm_provider": "openai",
}
with respx.mock:
respx.get(f"https://api.openai.com/v1/files/{_CLAIM_OUTPUT_FILE_ID}/content").mock(
return_value=httpx.Response(200, content=content)
)
instance: Final = self._instance(prisma, router)
await instance.check_batch_cost()
await instance.check_batch_cost()
assert row.batch_processed is True
assert row.status == "complete"
router.aretrieve_batch.assert_awaited_once()
assert len(callback.calls) == 1
payload: Final = callback.calls[0]["standard_logging_object"]
assert payload["model"] == expected_model
assert payload["response_cost"] == pytest.approx(successes * (100 * input_rate + 20 * output_rate))
assert payload["prompt_tokens"] == successes * 100
assert payload["completion_tokens"] == successes * 20
assert payload["hidden_params"]["batch_successful_requests"] == successes
assert payload["hidden_params"]["batch_failed_requests"] == failures
assert payload["hidden_params"]["batch_models"] == (
["batch-test-model"] if deployment_model == "openai/batch-test-model" else ["batch-test-model"] * successes
)
@pytest.mark.asyncio
@pytest.mark.parametrize("error_source", ("batch", "output"))
async def test_a_404_for_another_identifier_preserves_recovery(self, error_source: str) -> None:
import base64
import httpx
import litellm
import respx
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.utils import SpecialEnums
class SpendRecorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.costs: tuple[float, ...] = ()
async def async_log_success_event(
self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
) -> None:
if kwargs["call_type"] == "aretrieve_batch":
cost: Final = kwargs["response_cost"]
assert isinstance(cost, float)
self.costs += (cost,)
recorder: Final = SpendRecorder()
litellm.logging_callback_manager.add_litellm_async_success_callback(recorder)
litellm.register_model(
{
"batch-recovery-model": {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token_batches": 0.01,
"output_cost_per_token_batches": 0.02,
}
}
)
row: Final = _FakeManagedObjectRow()
row.unified_object_id = (
base64.urlsafe_b64encode(
SpecialEnums.LITELLM_MANAGED_BATCH_COMPLETE_STR.value.format("model-123", "batch-456").encode()
)
.decode()
.rstrip("=")
)
router: Final = self._router()
response: Final = router.aretrieve_batch.return_value
error: Final = litellm.NotFoundError(
message=f"No object found with id '{'batch-456' if error_source == 'batch' else _CLAIM_OUTPUT_FILE_ID}0'.",
model="gpt-4",
llm_provider="openai",
)
router.aretrieve_batch.side_effect = (error, response) if error_source == "batch" else (response, response)
prisma: Final = self._prisma(row, [])
output: Final = json.dumps(
{
"response": {
"status_code": 200,
"body": {
"model": "batch-recovery-model",
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
},
},
"error": None,
}
).encode()
with respx.mock:
route: Final = respx.get(f"https://api.openai.com/v1/files/{_CLAIM_OUTPUT_FILE_ID}/content")
route.mock(
side_effect=error if error_source == "output" else None,
return_value=None if error_source == "output" else httpx.Response(200, content=output),
)
await self._instance(prisma, router).check_batch_cost()
assert row.batch_processed is False
assert recorder.costs == ()
route.mock(return_value=httpx.Response(200, content=output))
await self._instance(prisma, router).check_batch_cost()
await self._instance(prisma, router).check_batch_cost()
assert row.batch_processed is True
assert row.status == "complete"
assert recorder.costs == (pytest.approx(10 * 0.01 + 5 * 0.02),)
@staticmethod
def _claim_calls(prisma) -> list:
return [
call.kwargs
for call in prisma.db.litellm_managedobjecttable.update_many.call_args_list
if "id" in call.kwargs["where"]
]
@staticmethod
async def _run_deletion_guard(prisma, file_id: str) -> None:
"""Run the real managed-files deletion guard against the row the poller is costing."""
from litellm_enterprise.proxy.hooks.managed_files import PROXY_LiteLLMManagedFiles
cache = MagicMock()
cache.async_get_cache = AsyncMock(return_value=None)
cache.async_set_cache = AsyncMock()
guard = PROXY_LiteLLMManagedFiles(internal_usage_cache=cache, prisma_client=prisma)
scheduler = MagicMock()
scheduler.get_job.return_value = MagicMock()
with patch("litellm.proxy.proxy_server.scheduler", scheduler):
await guard._check_file_deletion_allowed(file_id)
@pytest.mark.asyncio
@pytest.mark.parametrize("status", ("completed", "failed", "expired", "cancelled"))
@pytest.mark.parametrize("metadata_text", ("before\x00after", r"\u0000"))
async def test_cost_claim_persists_terminal_metadata_without_a_later_write(
self, status: str, metadata_text: str
) -> None:
row: Final = _FakeManagedObjectRow()
journal: Final = []
prisma: Final = self._prisma(row, journal)
router: Final = self._router()
router.aretrieve_batch.return_value = router.aretrieve_batch.return_value.model_copy(
update={"status": status, "metadata": {"annotation": metadata_text}}
)
prisma.db.litellm_managedobjecttable.update.side_effect = RuntimeError("final write unavailable")
worker: Final = self._instance(prisma, router)
with self._billing_patches(journal) as logging_obj:
await worker.check_batch_cost()
await worker.check_batch_cost()
assert row.batch_processed is True
assert row.status == ("complete" if status == "completed" else status)
assert json.loads(row.file_object)["status"] == status
assert json.loads(row.file_object)["metadata"] == {"annotation": metadata_text.replace("\x00", "")}
assert logging_obj.async_success_handler.await_count == 1
@pytest.mark.asyncio
async def test_winning_pod_claims_the_row_between_fetching_and_billing(self):
"""The claim flips batch_processed false -> true after the results are in hand and
before the spend log is written, so a concurrent pod's claim finds no matching row."""
row = _FakeManagedObjectRow()
journal = []
prisma = self._prisma(row, journal)
with self._billing_patches(journal) as logging_obj:
await self._instance(prisma, self._router()).check_batch_cost()
assert journal == ["fetch", "claim", "bill"]
assert row.status == "complete"
assert json.loads(row.file_object)["usage"] == {"prompt_tokens": 10, "completion_tokens": 5}
logging_obj.async_success_handler.assert_awaited_once()
assert row.batch_processed is True
@pytest.mark.asyncio
async def test_a_pod_that_loses_the_claim_after_fetching_does_not_bill(self):
"""Both pods select the row and fetch its results in the same window. The one whose
compare-and-swap finds the row already taken must not write a second spend log."""
row = _FakeManagedObjectRow()
journal = []
prisma = self._prisma(row, journal)
async def _other_pod_wins_the_row():
row.batch_processed = True
with self._billing_patches(journal, during_fetch=_other_pod_wins_the_row) as logging_obj:
await self._instance(prisma, self._router()).check_batch_cost()
assert journal == ["fetch"]
logging_obj.async_success_handler.assert_not_awaited()
assert row.status == "in_progress"
assert json.loads(row.file_object)["status"] == "in_progress"
@pytest.mark.asyncio
@pytest.mark.parametrize("stored_as_json_string", (True, False))
async def test_an_unclassified_logging_failure_keeps_the_claim(self, stored_as_json_string: bool) -> None:
row: Final = _FakeManagedObjectRow()
row.file_object = row.file_object if stored_as_json_string else json.loads(row.file_object)
journal: Final = []
prisma: Final = self._prisma(row, journal)
with self._billing_patches(journal, bill_error=Exception("Accounting outcome unknown")):
await self._instance(prisma, self._router()).check_batch_cost()
assert journal == ["fetch", "claim", "bill"]
assert row.batch_processed is True
with self._billing_patches(journal) as logging_obj:
await self._instance(prisma, self._router()).check_batch_cost()
await self._instance(prisma, self._router()).check_batch_cost()
logging_obj.async_success_handler.assert_not_awaited()
@pytest.mark.asyncio
async def test_a_worker_interrupted_mid_costing_leaves_the_batch_billable(self):
"""A pod killed while reading a batch's results must leave the row for a later
cycle. Claiming before the fetch marked the batch processed for good, so the pod
that died took that batch's spend with it and no other pod ever selected it."""
row = _FakeManagedObjectRow()
journal = []
prisma = self._prisma(row, journal)
reached_fetch = asyncio.Event()
async def _never_returns():
reached_fetch.set()
await asyncio.Event().wait()
with self._billing_patches(journal, during_fetch=_never_returns) as logging_obj:
interrupted = asyncio.create_task(self._instance(prisma, self._router()).check_batch_cost())
await asyncio.wait_for(reached_fetch.wait(), timeout=5)
assert row.batch_processed is False, "an in-flight costing must not mark the row processed"
interrupted.cancel()
with pytest.raises(asyncio.CancelledError):
await interrupted
assert journal == ["fetch"]
logging_obj.async_success_handler.assert_not_awaited()
survivor_journal = []
survivor_prisma = self._prisma(row, survivor_journal)
with self._billing_patches(survivor_journal) as survivor_logging:
await self._instance(survivor_prisma, self._router()).check_batch_cost()
assert survivor_journal == ["fetch", "claim", "bill"]
survivor_logging.async_success_handler.assert_awaited_once()
assert row.batch_processed is True
@pytest.mark.asyncio
async def test_costing_in_flight_keeps_the_referenced_file_undeletable(self):
"""The deletion guard only holds files whose batch still has batch_processed false,
so claiming the row before the fetch let a concurrent delete remove the very output
file the in-flight costing was about to read."""
row = _FakeManagedObjectRow()
journal = []
prisma = self._prisma(row, journal)
reached_fetch = asyncio.Event()
finish_fetch = asyncio.Event()
async def _wait_for_the_delete_attempt():
reached_fetch.set()
await finish_fetch.wait()
with self._billing_patches(journal, during_fetch=_wait_for_the_delete_attempt):
costing = asyncio.create_task(self._instance(prisma, self._router()).check_batch_cost())
await asyncio.wait_for(reached_fetch.wait(), timeout=5)
with pytest.raises(HTTPException) as blocked:
await self._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID)
assert blocked.value.status_code == 400
assert _CLAIM_OUTPUT_FILE_ID in blocked.value.detail
finish_fetch.set()
await asyncio.wait_for(costing, timeout=5)
assert journal == ["fetch", "claim", "bill"]
assert row.batch_processed is True
await self._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID)
@pytest.mark.asyncio
async def test_schema_without_batch_processed_still_bills(self):
"""Older schemas have no column to claim, so they keep the pre-fix behavior instead
of losing every batch's cost."""
row = _FakeManagedObjectRow()
journal = []
prisma = self._prisma(row, journal)
instance = self._instance(prisma, self._router())
instance._has_batch_processed_column = False
with self._billing_patches(journal) as logging_obj:
await instance.check_batch_cost()
assert self._claim_calls(prisma) == []
assert journal == ["fetch", "bill", "finalize"]
logging_obj.async_success_handler.assert_awaited_once()
class TestBatchLineItemsFromThePoller:
"""When batch_line_item_callbacks is set, the poller hands every line of a finished batch to those loggers
once the batch is billed and retired, in the background, so a slow, hung or failing sink never unbills, re-polls
or holds up a batch."""
_INPUT = b'{"custom_id":"ok-1","body":{"messages":[{"role":"user","content":"hi"}]}}\n'
_OUTPUT = (
b'{"custom_id":"ok-1","response":{"status_code":200,"body":{"model":"gpt-4",'
b'"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}}}\n'
)
_ERROR = b'{"custom_id":"ok-1","response":{"status_code":400,"body":{"error":{"message":"expired"}}}}\n'
@staticmethod
def _recording_logger(journal: list, error: Exception | None = None):
from litellm.integrations.custom_logger import CustomLogger
class _Recorder(CustomLogger):
def __init__(self):
super().__init__()
self.payloads = []
async def async_log_batch_line_items(self, payload):
journal.append("lines")
self.payloads.append(payload)
if error is not None:
raise error
return _Recorder()
@staticmethod
def _setup(journal: list, *, status="completed", output_file_id=_CLAIM_OUTPUT_FILE_ID, error_file_id=None):
row = _FakeManagedObjectRow()
prisma = TestMultiPodBatchCostClaim._prisma(row, journal)
router = TestMultiPodBatchCostClaim._router()
response = router.aretrieve_batch.return_value
response.status = status
response.output_file_id = output_file_id
response.error_file_id = error_file_id
response.input_file_id = "file-input-1"
return row, TestMultiPodBatchCostClaim._instance(prisma, router)
@staticmethod
async def _stored(instance) -> None:
"""Wait for the line items the poll handed to the background."""
await asyncio.gather(*instance._line_item_stores)
@classmethod
@contextmanager
def _provider(cls, journal: list):
"""The provider's batch files by id, and a spend-log writer that journals each write."""
files = {"file-input-1": cls._INPUT, _CLAIM_OUTPUT_FILE_ID: cls._OUTPUT, "file-err-1": cls._ERROR}
async def _afile_content(**kwargs):
journal.append(f"read {kwargs['file_id']}")
if kwargs["file_id"] not in files:
from litellm.exceptions import NotFoundError
raise NotFoundError(
message=f"No file found with id '{kwargs['file_id']}'.", model="gpt-4", llm_provider="openai"
)
content = MagicMock()
content.content = files[kwargs["file_id"]]
return content
def _is_b64(file_id):
if file_id == _CLAIM_UNIFIED_BATCH_ID:
return "llm_model_id,model-123;llm_batch_id,batch-456;"
return False
logging_obj = MagicMock()
logging_obj.async_success_handler = AsyncMock(side_effect=lambda **kwargs: journal.append("bill"))
with (
patch(_IS_B64, side_effect=_is_b64),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
return_value="model-123",
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
return_value="batch-456",
),
patch("litellm.files.main.afile_content", new=AsyncMock(side_effect=_afile_content)),
patch("litellm.litellm_core_utils.litellm_logging.Logging", return_value=logging_obj),
):
yield
@pytest.mark.asyncio
async def test_lines_reach_the_callbacks_after_the_claimed_batch_is_billed_and_retired(self, monkeypatch):
import litellm
journal: list = []
row, instance = self._setup(journal)
logger = self._recording_logger(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
with self._provider(journal):
await instance.check_batch_cost()
await self._stored(instance)
assert journal == [
f"read {_CLAIM_OUTPUT_FILE_ID}",
"claim",
"bill",
f"read {_CLAIM_OUTPUT_FILE_ID}",
"read file-input-1",
"lines",
]
(payload,) = logger.payloads
assert payload["batch_id"] == _CLAIM_UNIFIED_BATCH_ID
assert payload["finished_at"] == 2
assert payload["part"] == 0
(line,) = payload["lines"]
assert (line["custom_id"], line["status"]) == ("ok-1", "success")
assert line["usage"] == {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}
assert line["metadata"]["user_api_key_user_id"] == "user-1"
assert row.batch_processed is True
@pytest.mark.asyncio
async def test_the_lines_are_priced_off_the_event_loop(self, monkeypatch):
import threading
import litellm
from litellm.batches import batch_line_items
journal: list = []
row, instance = self._setup(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (self._recording_logger(journal),))
price = batch_line_items._line_item
threads = []
def _price(*args):
threads.append(threading.current_thread())
return price(*args)
monkeypatch.setattr(batch_line_items, "_line_item", _price)
with self._provider(journal):
await instance.check_batch_cost()
await self._stored(instance)
assert journal[-1] == "lines"
(thread,) = threads
assert thread is not threading.main_thread()
@pytest.mark.asyncio
async def test_a_creator_that_cannot_be_looked_up_gets_lines_without_bodies(self, monkeypatch):
import litellm
journal: list = []
row, instance = self._setup(journal)
row.api_key = sha256(b"sk-deleted-key").hexdigest()
logger = self._recording_logger(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
with self._provider(journal):
await instance.check_batch_cost()
await self._stored(instance)
(payload,) = logger.payloads
assert [(line["request"], line["response"]) for line in payload["lines"]] == [(None, None)]
assert "read file-input-1" not in journal
@pytest.mark.asyncio
async def test_a_creator_with_logging_on_gets_the_request_and_response(self, monkeypatch):
import litellm
journal: list = []
row, instance = self._setup(journal)
row.api_key = "cli-session-user-1"
logger = self._recording_logger(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
with self._provider(journal):
await instance.check_batch_cost()
await self._stored(instance)
assert journal[-4:] == ["bill", f"read {_CLAIM_OUTPUT_FILE_ID}", "read file-input-1", "lines"]
((line,),) = [payload["lines"] for payload in logger.payloads]
assert line["request"] == json.loads(self._INPUT)
assert line["response"] == json.loads(self._OUTPUT)
@pytest.mark.asyncio
async def test_a_failing_sink_leaves_the_batch_billed_and_retired(self, monkeypatch):
import litellm
journal: list = []
row, instance = self._setup(journal)
monkeypatch.setattr(
litellm, "batch_line_item_callbacks", (self._recording_logger(journal, RuntimeError("bucket gone")),)
)
with self._provider(journal):
await instance.check_batch_cost()
await self._stored(instance)
assert [step for step in journal if not step.startswith("read")] == ["claim", "bill", "lines"]
assert row.batch_processed is True
assert row.status == "complete"
@pytest.mark.asyncio
async def test_without_callbacks_only_the_output_file_is_read(self, monkeypatch):
import litellm
journal: list = []
_, instance = self._setup(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", ())
with self._provider(journal):
await instance.check_batch_cost()
await self._stored(instance)
assert journal == [f"read {_CLAIM_OUTPUT_FILE_ID}", "claim", "bill"]
@pytest.mark.asyncio
async def test_a_batch_where_every_line_failed_sends_its_error_lines_after_it_is_retired(self, monkeypatch):
import litellm
journal: list = []
row, instance = self._setup(journal, status="expired", output_file_id=None, error_file_id="file-err-1")
logger = self._recording_logger(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
with self._provider(journal):
await instance.check_batch_cost()
await self._stored(instance)
assert journal == ["finalize", "read file-err-1", "read file-input-1", "lines"]
(payload,) = logger.payloads
assert [(line["custom_id"], line["status"], line["response_cost"]) for line in payload["lines"]] == [
("ok-1", "failure", 0.0)
]
assert row.batch_processed is True
@pytest.mark.asyncio
async def test_a_batch_retired_with_nothing_billable_reads_its_input_file_by_the_id_the_provider_reported(
self, monkeypatch
):
import litellm
journal: list = []
_, instance = self._setup(journal, status="expired", output_file_id=None, error_file_id="file-err-1")
logger = self._recording_logger(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
async def _persist_managed_ids(*, response, **_):
response.input_file_id = "managed-id-of-the-copy-on-another-deployment"
with (
self._provider(journal),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.ensure_batch_response_managed_file_ids",
new=AsyncMock(side_effect=_persist_managed_ids),
),
):
await instance.check_batch_cost()
await self._stored(instance)
assert journal == ["finalize", "read file-err-1", "read file-input-1", "lines"]
((line,),) = [payload["lines"] for payload in logger.payloads]
assert line["request"] == json.loads(self._INPUT)
@pytest.mark.asyncio
async def test_a_batch_whose_files_hold_no_readable_line_sends_no_lines(self, monkeypatch):
import litellm
journal: list = []
_, instance = self._setup(journal, status="expired", output_file_id=None, error_file_id="file-err-1")
logger = self._recording_logger(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
monkeypatch.setattr(type(self), "_ERROR", b"not json\n")
with self._provider(journal):
await instance.check_batch_cost()
await self._stored(instance)
assert journal == ["finalize", "read file-err-1", "read file-input-1"]
assert logger.payloads == []
@pytest.mark.asyncio
@pytest.mark.parametrize("database_error", [False, True])
async def test_a_batch_that_could_not_be_retired_sends_no_lines_until_it_is(
self, monkeypatch: pytest.MonkeyPatch, database_error: bool
) -> None:
import litellm
journal: list = []
row, instance = self._setup(journal, status="expired", output_file_id=None, error_file_id="file-err-1")
instance.prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(
side_effect=RuntimeError("db down") if database_error else None, return_value=0
)
logger = self._recording_logger(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
with self._provider(journal):
await instance.check_batch_cost()
await self._stored(instance)
assert journal == []
assert logger.payloads == []
assert row.batch_processed is False
@pytest.mark.asyncio
@pytest.mark.parametrize("has_batch_processed_column, stored", [(True, 1), (False, 0)])
async def test_a_billed_batch_that_could_not_be_marked_complete_sends_lines_only_if_its_claim_retired_it(
self, monkeypatch, has_batch_processed_column, stored
):
import litellm
journal: list = []
row, instance = self._setup(journal)
instance._has_batch_processed_column = has_batch_processed_column
instance.prisma_client.db.litellm_managedobjecttable.update = AsyncMock(side_effect=RuntimeError("db down"))
logger = self._recording_logger(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
with self._provider(journal):
await instance.check_batch_cost()
await self._stored(instance)
assert journal.count("bill") == 1
assert len(logger.payloads) == stored
@pytest.mark.asyncio
async def test_a_batch_whose_output_file_is_gone_still_sends_its_error_lines_after_it_is_retired(self, monkeypatch):
import litellm
journal: list = []
row, instance = self._setup(journal, output_file_id="file-out-deleted", error_file_id="file-err-1")
logger = self._recording_logger(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
with self._provider(journal):
await instance.check_batch_cost()
await self._stored(instance)
assert journal == ["read file-out-deleted", "finalize", "read file-err-1", "read file-input-1", "lines"]
(payload,) = logger.payloads
assert [(line["custom_id"], line["status"]) for line in payload["lines"]] == [("ok-1", "failure")]
assert row.batch_processed is True
@pytest.mark.asyncio
async def test_a_terminal_batch_with_no_lines_sends_nothing(self, monkeypatch):
import litellm
journal: list = []
_, instance = self._setup(journal, status="failed", output_file_id=None, error_file_id=None)
logger = self._recording_logger(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
with self._provider(journal):
await instance.check_batch_cost()
await self._stored(instance)
assert journal == ["finalize"]
assert logger.payloads == []
@pytest.mark.asyncio
async def test_a_hung_sink_never_holds_up_billing_and_retiring_the_batch(self, monkeypatch):
import litellm
from litellm.integrations.custom_logger import CustomLogger
journal: list = []
row, instance = self._setup(journal)
released = asyncio.Event()
class _HungSink(CustomLogger):
async def async_log_batch_line_items(self, payload):
journal.append("lines started")
await released.wait()
journal.append("lines")
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (_HungSink(),))
async def _sink_started():
while "lines started" not in journal:
await asyncio.sleep(0)
with self._provider(journal):
await asyncio.wait_for(instance.check_batch_cost(), timeout=5)
await asyncio.wait_for(_sink_started(), timeout=5)
billed_while_hung = (row.batch_processed, row.status, list(journal))
released.set()
await self._stored(instance)
assert billed_while_hung == (
True,
"complete",
[
f"read {_CLAIM_OUTPUT_FILE_ID}",
"claim",
"bill",
f"read {_CLAIM_OUTPUT_FILE_ID}",
"read file-input-1",
"lines started",
],
)
assert journal[-1] == "lines"
@pytest.mark.asyncio
async def test_lines_are_dropped_with_an_error_while_a_poll_cycle_of_batches_is_still_waiting(
self, monkeypatch, caplog
):
import litellm
from litellm.constants import MAX_OBJECTS_PER_POLL_CYCLE
journal: list = []
row, instance = self._setup(journal)
logger = self._recording_logger(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
waiting = [asyncio.create_task(asyncio.sleep(3600)) for _ in range(MAX_OBJECTS_PER_POLL_CYCLE)]
instance._line_item_stores.update(waiting)
try:
with self._provider(journal), caplog.at_level(logging.ERROR):
await instance.check_batch_cost()
finally:
for task in waiting:
task.cancel()
assert logger.payloads == []
assert row.batch_processed is True
assert f"so the line items of batch batch-456 (job {row.id}) were not stored" in caplog.text
@pytest.mark.asyncio
async def test_batches_store_their_lines_one_at_a_time_in_the_order_they_were_retired(self, monkeypatch):
import litellm
journal: list = []
_, instance = self._setup(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (self._recording_logger(journal),))
running: list = []
async def _store(job, response, model_id, batch_id, *, billed):
running.append(batch_id)
journal.append(f"start {batch_id} with {len(running)} running")
await asyncio.sleep(0.01)
running.remove(batch_id)
monkeypatch.setattr(instance, "_store_batch_line_items", _store)
for batch_id in ("batch-a", "batch-b", "batch-c"):
instance._store_batch_line_items_in_background(MagicMock(), MagicMock(), "model-123", batch_id, billed=True)
await self._stored(instance)
assert journal == [f"start batch-{name} with 1 running" for name in "abc"]
@pytest.mark.asyncio
async def test_lines_are_stored_only_once_the_poll_has_billed_and_retired_every_batch_it_found(self, monkeypatch):
import litellm
journal: list = []
_, instance = self._setup(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (self._recording_logger(journal),))
async def _an_earlier_batch_waits_to_store_its_lines():
instance._store_batch_line_items_in_background(
MagicMock(), MagicMock(), "model-123", "batch-earlier", billed=True
)
await asyncio.sleep(0.01)
async def _store(job, response, model_id, batch_id, *, billed):
journal.append(f"store {batch_id}")
monkeypatch.setattr(instance, "_cleanup_stale_managed_objects", _an_earlier_batch_waits_to_store_its_lines)
monkeypatch.setattr(instance, "_store_batch_line_items", _store)
with self._provider(journal):
await instance.check_batch_cost()
await self._stored(instance)
assert journal == [
f"read {_CLAIM_OUTPUT_FILE_ID}",
"claim",
"bill",
"store batch-earlier",
"store batch-456",
]
@pytest.mark.asyncio
async def test_an_output_file_that_cannot_be_read_again_leaves_the_batch_billed_and_stores_nothing(
self, monkeypatch, caplog
):
import litellm
from litellm.batches import batch_utils
from litellm.exceptions import NotFoundError
journal: list = []
row, instance = self._setup(journal)
logger = self._recording_logger(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
fetch = batch_utils.fetch_batch_managed_file_content
async def _output_gone_after_billing(file_id, **kwargs):
if file_id == _CLAIM_OUTPUT_FILE_ID:
raise NotFoundError(message="No file found.", model="gpt-4", llm_provider="openai")
return await fetch(file_id, **kwargs)
monkeypatch.setattr(batch_utils, "fetch_batch_managed_file_content", _output_gone_after_billing)
with self._provider(journal), caplog.at_level(logging.ERROR):
await instance.check_batch_cost()
await self._stored(instance)
assert journal == [f"read {_CLAIM_OUTPUT_FILE_ID}", "claim", "bill"]
assert logger.payloads == []
assert (row.batch_processed, row.status) == (True, "complete")
assert f"the line items of batch batch-456 (job {row.id}) were not stored" in caplog.text
@pytest.mark.asyncio
async def test_a_deployment_removed_before_the_lines_are_stored_leaves_the_batch_billed_and_logs_an_error(
self, monkeypatch, caplog
):
import litellm
journal: list = []
row, instance = self._setup(journal)
logger = self._recording_logger(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
deployment = instance.llm_router.get_deployment.return_value
instance.llm_router.get_deployment = MagicMock(
side_effect=lambda **_: None if instance._poll_idle.is_set() else deployment
)
with self._provider(journal), caplog.at_level(logging.ERROR):
await instance.check_batch_cost()
await self._stored(instance)
assert journal == [f"read {_CLAIM_OUTPUT_FILE_ID}", "claim", "bill"]
assert logger.payloads == []
assert (row.batch_processed, row.status) == (True, "complete")
assert f"the line items of batch batch-456 (job {row.id}) were not stored" in caplog.text
@pytest.mark.asyncio
async def test_a_batch_bigger_than_one_part_reaches_the_callbacks_as_numbered_parts(self, monkeypatch):
import litellm
from litellm_enterprise.proxy.common_utils import check_batch_cost
journal: list = []
_, instance = self._setup(journal)
logger = self._recording_logger(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
monkeypatch.setattr(check_batch_cost, "BATCH_LINE_ITEMS_MAX_PART_BYTES", 1)
monkeypatch.setattr(type(self), "_INPUT", self._INPUT + self._INPUT.replace(b"ok-1", b"ok-2"))
monkeypatch.setattr(type(self), "_OUTPUT", self._OUTPUT + self._OUTPUT.replace(b"ok-1", b"ok-2"))
with self._provider(journal):
await instance.check_batch_cost()
await self._stored(instance)
assert [(payload["part"], [line["custom_id"] for line in payload["lines"]]) for payload in logger.payloads] == [
(0, ["ok-1"]),
(1, ["ok-2"]),
]
assert [payload["lines"][0]["request"]["custom_id"] for payload in logger.payloads] == ["ok-1", "ok-2"]
@pytest.mark.asyncio
async def test_an_oauth2_creators_token_reaches_the_callbacks_only_as_its_hash(self, monkeypatch):
import litellm
journal: list = []
row, instance = self._setup(journal)
row.api_key = "ya29.opaque-oauth2-access-token"
logger = self._recording_logger(journal)
monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
with self._provider(journal):
await instance.check_batch_cost()
await self._stored(instance)
(payload,) = logger.payloads
(line,) = payload["lines"]
assert line["metadata"]["user_api_key_hash"] == sha256(b"ya29.opaque-oauth2-access-token").hexdigest()
assert line["request"] == json.loads(self._INPUT)
assert "ya29.opaque-oauth2-access-token" not in json.dumps(payload)
class TestCreatorTurnedOffMessageLogging:
"""Batch line items drop the request and response when the creating key or team turned off message logging,
when the proxy did, or when the creator can no longer be read."""
@staticmethod
def _instance(key_row=None, team_row=None, error: Exception | None = None):
from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost
prisma = MagicMock()
prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=key_row, side_effect=error)
prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=team_row)
return CheckBatchCost(proxy_logging_obj=MagicMock(), prisma_client=prisma, llm_router=MagicMock())
@staticmethod
def _job(api_key=sha256(b"sk-alice").hexdigest(), team_id=None, created_by="alice"):
from types import SimpleNamespace
return SimpleNamespace(id="job-1", api_key=api_key, team_id=team_id, created_by=created_by)
@staticmethod
def _logging(value: str) -> dict:
return {
"logging": [
{
"callback_name": "s3_v2",
"callback_type": "success",
"callback_vars": {"turn_off_message_logging": value},
}
]
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("key_value", "proxy_wide", "expected"),
[("true", False, True), ("false", True, False), (None, True, True), (None, False, False)],
)
async def test_the_key_setting_wins_and_the_proxy_setting_fills_in(
self, monkeypatch, key_value, proxy_wide, expected
):
from types import SimpleNamespace
import litellm
monkeypatch.setattr(litellm, "turn_off_message_logging", proxy_wide)
metadata = self._logging(key_value) if key_value is not None else {}
instance = self._instance(key_row=SimpleNamespace(metadata=metadata))
assert await instance._creator_turned_off_message_logging(self._job()) is expected
@pytest.mark.asyncio
async def test_a_team_that_turned_off_message_logging_covers_its_keys(self, monkeypatch):
from types import SimpleNamespace
import litellm
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
instance = self._instance(
key_row=SimpleNamespace(metadata={}), team_row=SimpleNamespace(metadata=self._logging("true"))
)
assert await instance._creator_turned_off_message_logging(self._job(team_id="team-alpha")) is True
@pytest.mark.asyncio
@pytest.mark.parametrize(
("job_fields", "key_row", "team_row"),
[
({}, None, None),
({"team_id": "team-gone"}, "present", None),
({"api_key": None, "team_id": "team-gone"}, None, None),
],
ids=["deleted-key", "deleted-team", "deleted-team-of-a-batch-created-without-a-key"],
)
async def test_a_creator_that_is_gone_counts_as_turned_off(self, monkeypatch, job_fields, key_row, team_row):
from types import SimpleNamespace
import litellm
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
instance = self._instance(key_row=SimpleNamespace(metadata={}) if key_row else None, team_row=team_row)
assert await instance._creator_turned_off_message_logging(self._job(**job_fields)) is True
@pytest.mark.asyncio
@pytest.mark.parametrize("api_key", [None, "opaque-oauth2-access-token"], ids=["jwt", "oauth2"])
@pytest.mark.parametrize(
("team_value", "proxy_wide", "expected"),
[(None, False, False), (None, True, True), ("true", False, True), ("false", True, False)],
)
async def test_a_batch_created_without_a_virtual_key_follows_its_team_and_the_proxy_setting(
self, monkeypatch, api_key, team_value, proxy_wide, expected
):
from types import SimpleNamespace
import litellm
monkeypatch.setattr(litellm, "turn_off_message_logging", proxy_wide)
team_row = SimpleNamespace(metadata=self._logging(team_value)) if team_value is not None else None
instance = self._instance(team_row=team_row)
job = self._job(api_key=api_key, team_id="team-alpha" if team_row is not None else None, created_by="alice")
assert await instance._creator_turned_off_message_logging(job) is expected
instance.prisma_client.db.litellm_verificationtoken.find_unique.assert_not_called()
@pytest.mark.asyncio
async def test_a_failed_lookup_counts_as_turned_off(self, monkeypatch):
import litellm
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
instance = self._instance(error=Exception("db down"))
assert await instance._creator_turned_off_message_logging(self._job()) is True
@pytest.mark.asyncio
@pytest.mark.parametrize("api_key", ["litellm_proxy_master_key", "cli-session-alice"])
async def test_the_master_key_and_cli_sessions_have_no_key_row_to_read(self, monkeypatch, api_key):
import litellm
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
instance = self._instance()
assert await instance._creator_turned_off_message_logging(self._job(api_key=api_key)) is False
instance.prisma_client.db.litellm_verificationtoken.find_unique.assert_not_called()
@pytest.mark.asyncio
async def test_old_completed_batch_remains_available_for_costing() -> None:
row: Final = _FakeManagedObjectRow()
row.status = "completed"
row.created_at = datetime(2000, 1, 1, tzinfo=timezone.utc)
prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router())
await poller._cleanup_stale_managed_objects()
assert row.batch_processed is False
assert row.status == "completed"
@pytest.mark.asyncio
@pytest.mark.parametrize("progress", ["claimed", "refreshed"])
async def test_stale_retirement_preserves_concurrent_progress(progress: str) -> None:
row: Final = _FakeManagedObjectRow()
selected: Final = copy(row)
if progress == "claimed":
row.batch_processed = True
row.updated_at = datetime(2000, 1, 2, tzinfo=timezone.utc)
prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router())
await poller._retire_job(selected, "provider batch disappeared")
assert row.batch_processed is (progress == "claimed")
assert row.updated_at > selected.updated_at
@pytest.mark.asyncio
@pytest.mark.parametrize("progress", ["claimed", "refreshed"])
async def test_stale_terminal_response_preserves_concurrent_progress(progress: str) -> None:
from litellm.types.utils import LiteLLMBatch
row: Final = _FakeManagedObjectRow()
selected: Final = copy(row)
if progress == "claimed":
row.batch_processed = True
row.updated_at = datetime(2000, 1, 2, tzinfo=timezone.utc)
row.file_object = '{"revision":"fresh"}'
prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router())
response: Final = LiteLLMBatch(
id="batch-456",
completion_window="24h",
created_at=1,
endpoint="/v1/chat/completions",
input_file_id="file-input-1",
object="batch",
status="failed",
)
assert await poller._finalize_unbilled_terminal_job(selected, response) is False
assert row.file_object == '{"revision":"fresh"}'
assert row.status == "in_progress"
assert row.batch_processed is (progress == "claimed")
@pytest.mark.asyncio
@pytest.mark.parametrize("legacy", [False, True])
async def test_permanently_lost_batch_retires_unchanged_row(legacy: bool) -> None:
row: Final = _FakeManagedObjectRow()
prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router())
poller._has_batch_processed_column = not legacy
await poller._retire_job(row, "provider batch disappeared")
assert row.batch_processed is (not legacy)
assert row.status == ("stale_expired" if legacy else "in_progress")
@pytest.mark.asyncio
async def test_failed_retirement_keeps_batch_available() -> None:
row: Final = _FakeManagedObjectRow()
prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
prisma.db.litellm_managedobjecttable.update_many = AsyncMock(side_effect=RuntimeError("database unavailable"))
poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router())
await poller._retire_job(row, "provider batch disappeared")
assert row.batch_processed is False
assert row.status == "in_progress"
@pytest.mark.parametrize("resource_id", ("batch-123", "file-123", "gs://bucket/job/predictions.jsonl"))
@pytest.mark.parametrize("other_id", ("{id}4", "prefix-{id}", "{id}/child", "{id}.backup", "{id}?generation=2"))
def test_another_resource_in_a_404_is_not_permanent_loss(resource_id: str, other_id: str) -> None:
from litellm.exceptions import NotFoundError
from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost
error: Final = NotFoundError(
message=f"No object found with id '{other_id.format(id=resource_id)}'.", model="test", llm_provider="openai"
)
assert CheckBatchCost._is_batch_gone_at_provider(error, resource_id) is False
assert CheckBatchCost._is_output_file_gone_at_provider(error, resource_id) is False
@pytest.mark.parametrize("resource_id", ("batch-123", "file-123", "gs://bucket/job/predictions.jsonl"))
@pytest.mark.parametrize("message", ("No object found with id '{id}'.", 'File "{id}" does not exist', "{id} not found"))
def test_a_404_naming_the_exact_resource_is_permanent_loss(resource_id: str, message: str) -> None:
from litellm.exceptions import NotFoundError
from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost
error: Final = NotFoundError(message=message.format(id=resource_id), model="test", llm_provider="openai")
assert CheckBatchCost._is_batch_gone_at_provider(error, resource_id) is True
assert CheckBatchCost._is_output_file_gone_at_provider(error, resource_id) is True
@pytest.mark.parametrize("output_file_id", (None, ""))
def test_a_404_without_an_output_identity_remains_retryable(output_file_id: str | None) -> None:
from litellm.exceptions import NotFoundError
from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost
error: Final = NotFoundError(message="Output file not found", model="test", llm_provider="openai")
assert CheckBatchCost._is_output_file_gone_at_provider(error, output_file_id) is False
@pytest.mark.asyncio
@pytest.mark.parametrize(
"kind", ["sync", "async", "custom", "overlapping", "general", "mixed", "sync_only", "executor_shutdown"]
)
@pytest.mark.parametrize("initialized", [False, True])
@pytest.mark.parametrize("claimed", [False, True])
async def test_batch_cost_reaches_configured_success_callbacks(
kind: Literal["sync", "async", "custom", "overlapping", "general", "mixed", "sync_only", "executor_shutdown"],
initialized: bool,
claimed: bool,
) -> None:
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.types.utils import LiteLLMBatch
from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost
events: Final[SimpleQueue[tuple[str, object, int, int]]] = SimpleQueue()
def record(
kwargs: dict[str, object],
response_obj: object,
start_time: datetime,
end_time: datetime,
recipient: str = "function",
) -> None:
if kwargs.get("call_type") != "aretrieve_batch":
return
assert isinstance(response_obj, LiteLLMBatch)
assert response_obj.usage is not None
events.put(
(recipient, kwargs["response_cost"], response_obj.usage.prompt_tokens, response_obj.usage.completion_tokens)
)
async def async_record(
kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
) -> None:
record(kwargs, response_obj, start_time, end_time)
if kind == "executor_shutdown" and kwargs.get("call_type") == "aretrieve_batch":
executor.shutdown(wait=True)
class BatchRecorder(CustomLogger):
def log_success_event(
self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
) -> None:
record(kwargs, response_obj, start_time, end_time, "custom")
async def async_log_success_event(
self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
) -> None:
record(kwargs, response_obj, start_time, end_time, "custom")
class SyncRecorder(CustomLogger):
def log_success_event(
self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
) -> None:
record(kwargs, response_obj, start_time, end_time, "custom")
callback: Final = SyncRecorder() if kind == "sync_only" else BatchRecorder()
match kind:
case "sync":
litellm.logging_callback_manager.add_litellm_success_callback(record)
case "async":
litellm.logging_callback_manager.add_litellm_success_callback(async_record)
case "executor_shutdown":
litellm.logging_callback_manager.add_litellm_success_callback(async_record)
litellm.logging_callback_manager.add_litellm_success_callback(record)
case "custom" | "overlapping" | "sync_only":
litellm.logging_callback_manager.add_litellm_success_callback(callback)
if kind == "overlapping":
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
case "general":
litellm.logging_callback_manager.add_litellm_callback(callback)
case "mixed":
litellm.logging_callback_manager.add_litellm_success_callback(callback)
litellm.logging_callback_manager.add_litellm_success_callback(record)
model: Final = "gpt-6.1-sol"
credentials: Final = {
"api_key": "synthetic-key",
"api_base": "https://batch.invalid/v1",
"custom_llm_provider": "openai",
}
deployment: Final = SimpleNamespace(
litellm_params=LiteLLM_Params(model=f"openai/{model}", custom_llm_provider="openai", api_base=None),
model_name="batch-group",
)
prisma: Final = MagicMock()
prisma.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=int(claimed))
checker: Final = CheckBatchCost(
proxy_logging_obj=SimpleNamespace(get_proxy_hook=lambda _: None),
prisma_client=prisma,
llm_router=SimpleNamespace(
get_deployment_credentials_with_provider=lambda _: credentials,
get_deployment=lambda **_: deployment,
),
)
job: Final = SimpleNamespace(
id="job",
unified_object_id="batch-record",
status="in_progress",
file_object={},
created_by=None,
team_id=None,
api_key=None,
org_id=None,
request_tags=None,
)
response: Final = LiteLLMBatch(
id="batch-record",
completion_window="24h",
created_at=1,
endpoint="/v1/chat/completions",
input_file_id="input",
object="batch",
status="completed",
output_file_id="output",
)
output: Final = json.dumps(
{
"id": "line",
"response": {
"status_code": 200,
"body": {
"model": model,
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
"choices": [
{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}
],
},
},
}
).encode()
with (
ThreadPoolExecutor(max_workers=1) as executor,
patch("litellm.litellm_core_utils.litellm_logging.executor", executor),
respx.mock(assert_all_called=True) as transport,
):
transport.get("https://batch.invalid/v1/files/output/content").respond(200, content=output)
if initialized:
await litellm.afile_content(file_id="output", **credentials)
await GLOBAL_LOGGING_WORKER.flush()
result: Final = await checker._track_completed_batch_cost(job, response, "deployment", "batch-record", None)
assert result == ((model, "openai") if claimed else None)
if kind == "executor_shutdown" and claimed:
prisma.db.litellm_managedobjecttable.update_many.assert_awaited_once()
await GLOBAL_LOGGING_WORKER.flush()
recipients: Final = (
("custom", "function")
if kind == "mixed"
else ("custom",)
if kind in ("custom", "overlapping", "general", "sync_only")
else ("function",)
)
expected: Final = recipients if claimed else ()
assert events.qsize() == len(expected)
received: Final = tuple(events.get_nowait() for _ in expected)
assert tuple(sorted(event[0] for event in received)) == expected
for _, cost, prompt_tokens, completion_tokens in received:
assert isinstance(cost, float) and cost > 0
assert (prompt_tokens, completion_tokens) == (10, 5)