Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

199 KB
Raw Blame
"""
Unit tests for CheckBatchCost class.
Covers: stale-row cleanup (file_purpose scoping), paginated find_many,
the batch_processed-column fallback query, and routing of unmanaged
Vertex (raw gs:// input_file_id) and Bedrock (raw s3:// input_file_id,
ARN unified_object_id) batches with no managed unified id.
"""

import asyncio
import json
import logging
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from copy import copy
from datetime import datetime, timezone
from hashlib import sha256
from queue import SimpleQueue
from types import SimpleNamespace
from typing import TYPE_CHECKING, Final, Literal
from unittest.mock import AsyncMock, MagicMock, patch

import pytest
import respx
from fastapi import HTTPException

from litellm.types.router import LiteLLM_Params
from litellm.types.utils import LiteLLMBatch

if TYPE_CHECKING:
    from litellm.batches.batch_utils import BatchCostUsageResult

_IS_B64 = "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id"
_CLAIM_UNIFIED_BATCH_ID = "dW5pZmllZF9iYXRjaF9pZA=="
_CLAIM_OUTPUT_FILE_ID = "file-output-123"


def test_batch_output_file_object_derives_metadata():
    from litellm_enterprise.proxy.hooks.managed_files import _batch_output_file_object

    output_file = _batch_output_file_object(
        unified_file_id="unified-output",
        raw_file_id="s3://bucket/path/to/output.jsonl.out",
        size_bytes=4321,
        fallback=False,
    )
    provider_file = _batch_output_file_object(
        unified_file_id="unified-provider",
        raw_file_id="file-abc",
        size_bytes=0,
        fallback=False,
    )

    assert output_file.id == "unified-output"
    assert output_file.object == "file"
    assert output_file.purpose == "batch_output"
    assert output_file.filename == "output.jsonl.out"
    assert output_file.bytes == 4321
    assert output_file.status == "processed"
    assert provider_file.filename == "file-abc"
    assert provider_file.bytes == 0


def _batch_cost_result(
    cost: float,
    usage: dict,
    models: list[str],
    successful_requests: int = 1,
    failed_requests: int = 0,
) -> "BatchCostUsageResult":
    """Build the BatchCostUsageResult calculate_batch_cost_and_usage now returns,
    for mocking it in tests that only care about cost/usage/models."""
    from litellm.batches.batch_utils import BatchCostUsageResult

    return BatchCostUsageResult(
        cost=cost,
        usage=usage,
        models=models,
        successful_requests=successful_requests,
        failed_requests=failed_requests,
    )


def _unmanaged_vertex_file_object(
    input_file_id="gs://bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash/abc.jsonl",
    status="validating",
):
    """A LiteLLMBatch JSON blob shaped like what the managed-files hook stores for an
    unmanaged Vertex batch (raw gs:// input_file_id)."""
    from litellm.types.utils import LiteLLMBatch

    return LiteLLMBatch(
        id="8823717160934178816",
        completion_window="24h",
        created_at=1,
        endpoint="/v1/chat/completions",
        input_file_id=input_file_id,
        object="batch",
        status=status,
    ).model_dump_json()


def _unmanaged_bedrock_file_object(
    input_file_id=(
        "s3://bucket/litellm-bedrock-files-us.anthropic.claude-sonnet-4-20250514-v1-0"
        "-74b61828-9191-4d80-addb-5a0f9ab0ec6a.jsonl"
    ),
    status="validating",
):
    """A LiteLLMBatch JSON blob shaped like what gets stored for an unmanaged Bedrock
    batch (raw s3:// input_file_id, ARN unified_object_id)."""
    from litellm.types.utils import LiteLLMBatch

    return LiteLLMBatch(
        id="arn:aws:bedrock:us-east-1:298249409318:model-invocation-job/1ofb47x17jua",
        completion_window="24h",
        created_at=1,
        endpoint="/v1/chat/completions",
        input_file_id=input_file_id,
        object="batch",
        status=status,
    ).model_dump_json()


class TestCheckBatchCost:
    """Test suite for CheckBatchCost class"""

    @pytest.fixture
    def mock_prisma_client(self):
        client = MagicMock()
        client.db = MagicMock()
        client.db.litellm_managedobjecttable = MagicMock()
        client.db.litellm_usertable = MagicMock()
        return client

    @pytest.fixture
    def mock_proxy_logging_obj(self):
        mock = MagicMock()
        mock.get_proxy_hook.return_value = None
        return mock

    @pytest.fixture
    def mock_llm_router(self):
        return MagicMock()

    @pytest.fixture
    def check_batch_cost_instance(self, mock_proxy_logging_obj, mock_prisma_client, mock_llm_router):
        from litellm_enterprise.proxy.common_utils.check_batch_cost import (
            CheckBatchCost,
        )

        return CheckBatchCost(
            proxy_logging_obj=mock_proxy_logging_obj,
            prisma_client=mock_prisma_client,
            llm_router=mock_llm_router,
        )

    @pytest.mark.asyncio
    async def test_cleanup_scoped_to_batch_file_purpose(self, check_batch_cost_instance, mock_prisma_client):
        """_cleanup_stale_managed_objects scopes its update to file_purpose='batch' only."""
        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        # Return empty so the main poll loop exits immediately
        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[])

        await check_batch_cost_instance.check_batch_cost()

        calls = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args_list
        stale_call = calls[0]
        assert stale_call[1]["data"] == {"status": "stale_expired"}
        where = stale_call[1]["where"]
        assert where["file_purpose"] == "batch"
        assert "stale_expired" in where["status"]["not_in"]
        assert "created_at" in where

    @pytest.mark.asyncio
    async def test_startup_probe_confirms_batch_processed_support(self, check_batch_cost_instance, mock_prisma_client):
        mock_prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=None)

        await check_batch_cost_instance.confirm_batch_processed_support()

        probe_where = mock_prisma_client.db.litellm_managedobjecttable.find_first.call_args[1]["where"]
        assert probe_where["batch_processed"] is False
        assert check_batch_cost_instance.batch_processed_support_confirmed is True
        assert check_batch_cost_instance._has_batch_processed_column is True

    @pytest.mark.asyncio
    async def test_startup_probe_marks_column_absent(self, check_batch_cost_instance, mock_prisma_client):
        mock_prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock(
            side_effect=Exception("column batch_processed does not exist")
        )

        await check_batch_cost_instance.confirm_batch_processed_support()

        assert check_batch_cost_instance.batch_processed_support_confirmed is False
        assert check_batch_cost_instance._has_batch_processed_column is False

    @pytest.mark.asyncio
    async def test_startup_probe_transient_error_defers_to_poll_cycle(
        self, check_batch_cost_instance, mock_prisma_client
    ):
        mock_prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock(
            side_effect=Exception("connection reset by peer")
        )

        await check_batch_cost_instance.confirm_batch_processed_support()

        assert check_batch_cost_instance.batch_processed_support_confirmed is False
        assert check_batch_cost_instance._has_batch_processed_column is True

    @pytest.mark.asyncio
    async def test_find_many_uses_pagination_and_excludes_stale(self, check_batch_cost_instance, mock_prisma_client):
        """find_many is called with take, order, and all terminal statuses excluded."""
        from litellm.constants import MAX_OBJECTS_PER_POLL_CYCLE

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[])

        await check_batch_cost_instance.check_batch_cost()

        find_call = mock_prisma_client.db.litellm_managedobjecttable.find_many.call_args
        assert find_call[1]["take"] == MAX_OBJECTS_PER_POLL_CYCLE
        assert find_call[1]["order"] == [{"created_at": "asc"}, {"id": "asc"}]
        not_in = find_call[1]["where"]["status"]["not_in"]
        assert "stale_expired" in not_in
        # "complete"/"completed" are intentionally NOT excluded from the
        # primary query — the batch_processed=False filter is sufficient.
        # This allows CheckBatchCost to pick up batches that were
        # transitioned to "complete" by the retrieve_batch endpoint
        # before CheckBatchCost had a chance to process them.
        assert "complete" not in not_in
        assert "completed" not in not_in
        assert find_call[1]["where"]["batch_processed"] is False
        assert check_batch_cost_instance.batch_processed_support_confirmed is True

    @pytest.mark.asyncio
    async def test_fallback_query_used_when_batch_processed_missing(
        self, check_batch_cost_instance, mock_prisma_client
    ):
        """Falls back to query without batch_processed when primary query raises."""
        from litellm.constants import MAX_OBJECTS_PER_POLL_CYCLE

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        # First find_many (primary query) raises with a schema error; second (fallback) returns empty
        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(
            side_effect=[Exception("column batch_processed does not exist"), [], []]
        )

        await check_batch_cost_instance.check_batch_cost()

        calls = mock_prisma_client.db.litellm_managedobjecttable.find_many.call_args_list
        assert len(calls) == 3
        fallback_where = calls[1][1]["where"]
        assert "batch_processed" not in fallback_where
        assert "stale_expired" in fallback_where["status"]["not_in"]
        assert calls[1][1]["take"] == MAX_OBJECTS_PER_POLL_CYCLE
        # Column absence is now cached — next call should go straight to fallback
        assert check_batch_cost_instance._has_batch_processed_column is False
        assert check_batch_cost_instance.batch_processed_support_confirmed is False

    @pytest.mark.asyncio
    async def test_column_absence_cached_across_cycles(self, check_batch_cost_instance, mock_prisma_client):
        """After column absence is discovered, subsequent cycles skip the primary query entirely."""
        from litellm.constants import MAX_OBJECTS_PER_POLL_CYCLE

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        # Simulate column already known absent from a previous cycle
        check_batch_cost_instance._has_batch_processed_column = False
        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[])

        await check_batch_cost_instance.check_batch_cost()

        assert mock_prisma_client.db.litellm_managedobjecttable.find_many.call_count == 2
        fallback_where = mock_prisma_client.db.litellm_managedobjecttable.find_many.call_args[1]["where"]
        assert "batch_processed" not in fallback_where

    @pytest.mark.asyncio
    async def test_fallback_completion_update_omits_batch_processed(
        self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
    ):
        """When batch_processed column is absent, completion update must not include it.

        If it did, the update would fail silently, the job would never be marked done,
        and every subsequent poll cycle would re-log the cost (duplicate billing).
        """
        from unittest.mock import patch

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
        mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        mock_job = MagicMock()
        mock_job.id = "job-fallback-1"
        mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
        mock_job.created_by = "user-1"

        # Simulate column already known absent (e.g. discovered on a previous cycle)
        check_batch_cost_instance._has_batch_processed_column = False
        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])

        # Build a fake batch response whose status triggers the completion branch
        mock_response = MagicMock()
        mock_response.status = "completed"
        mock_response.output_file_id = "file-output-123"
        mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"completed"}'

        mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
        mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})

        mock_deployment = MagicMock()
        mock_deployment.litellm_params.custom_llm_provider = "openai"
        mock_deployment.litellm_params.model = "gpt-4"
        mock_deployment.model_info.model_dump.return_value = {}
        mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)

        mock_file_content = MagicMock()
        mock_file_content.content = b'{"id":"req-1"}'

        decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;"

        with (
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
                side_effect=[decoded_id, None],
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
                return_value="model-123",
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
                return_value="batch-456",
            ),
            patch(
                "litellm.files.main.afile_content",
                new_callable=AsyncMock,
                return_value=mock_file_content,
            ),
            patch(
                "litellm.batches.batch_utils.get_file_content_as_dictionary",
                return_value=[{"id": "req-1"}],
            ),
            patch(
                "litellm.batches.batch_utils.calculate_batch_cost_and_usage",
                new_callable=AsyncMock,
                return_value=_batch_cost_result(
                    0.01,
                    {"prompt_tokens": 10, "completion_tokens": 5},
                    ["gpt-4"],
                ),
            ),
            patch(
                "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
                return_value=("gpt-4", "openai", None, None),
            ),
            patch("litellm.litellm_core_utils.litellm_logging.Logging") as mock_logging_cls,
        ):
            mock_logging_obj = MagicMock()
            mock_logging_obj.async_success_handler = AsyncMock()
            mock_logging_cls.return_value = mock_logging_obj

            await check_batch_cost_instance.check_batch_cost()

        # The update must have been called — this is the core assertion.
        assert mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 1, (
            "Expected update() to be called exactly once for the completed job"
        )
        update_data = mock_prisma_client.db.litellm_managedobjecttable.update.call_args[1]["data"]
        assert "batch_processed" not in update_data, "update() must NOT include batch_processed when column is absent"
        assert update_data["status"] == "complete"

    @pytest.mark.asyncio
    async def test_output_fetch_passes_deployment_credentials_as_trusted_snapshot(
        self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
    ):
        """Bedrock resolves the output bucket ONLY from the immutable snapshot kwarg.

        Spreading the credentials as plain kwargs is not enough: get_litellm_params drops
        s3_bucket_name, so without _litellm_internal_model_credentials the cost poller
        cannot read the output file and every completed Bedrock batch stays unbilled.
        """
        from types import MappingProxyType
        from unittest.mock import patch

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
        mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        mock_job = MagicMock()
        mock_job.id = "job-bedrock-1"
        mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
        mock_job.created_by = "user-1"
        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])

        mock_response = MagicMock()
        mock_response.status = "completed"
        mock_response.output_file_id = "file-output-123"
        mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"completed"}'

        mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
        mock_llm_router.get_deployment_credentials_with_provider = MagicMock(
            return_value={
                "custom_llm_provider": "bedrock",
                "s3_bucket_name": "configured-batch-bucket",
                "aws_region_name": "us-east-1",
            }
        )

        mock_deployment = MagicMock()
        mock_deployment.litellm_params.custom_llm_provider = "bedrock"
        mock_deployment.litellm_params.model = "bedrock/anthropic.claude-haiku-4-5-20251001-v1:0"
        mock_deployment.model_info.model_dump.return_value = {}
        mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)

        mock_file_content = MagicMock()
        mock_file_content.content = b'{"recordId":"req-1"}'

        decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;"

        with (
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
                side_effect=[decoded_id, None],
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
                return_value="model-123",
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
                return_value="batch-456",
            ),
            patch(
                "litellm.files.main.afile_content",
                new_callable=AsyncMock,
                return_value=mock_file_content,
            ) as mock_afile_content,
            patch(
                "litellm.batches.batch_utils.get_file_content_as_dictionary",
                return_value=[{"recordId": "req-1"}],
            ),
            patch(
                "litellm.batches.batch_utils.calculate_batch_cost_and_usage",
                new_callable=AsyncMock,
                return_value=_batch_cost_result(
                    0.01, {"prompt_tokens": 10, "completion_tokens": 5}, ["claude-haiku-4-5"]
                ),
            ),
            patch(
                "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
                return_value=("anthropic.claude-haiku-4-5-20251001-v1:0", "bedrock", None, None),
            ),
            patch("litellm.litellm_core_utils.litellm_logging.Logging") as mock_logging_cls,
        ):
            mock_logging_obj = MagicMock()
            mock_logging_obj.async_success_handler = AsyncMock()
            mock_logging_cls.return_value = mock_logging_obj

            await check_batch_cost_instance.check_batch_cost()

        mock_afile_content.assert_awaited()
        passed_kwargs = mock_afile_content.await_args[1]
        snapshot = passed_kwargs.get("_litellm_internal_model_credentials")
        assert snapshot is not None, "cost poller must pass the trusted credential snapshot"
        assert isinstance(snapshot, MappingProxyType), (
            "snapshot must be a MappingProxyType; a plain dict is rejected by get_configured_s3_bucket_name"
        )
        assert snapshot["s3_bucket_name"] == "configured-batch-bucket"

    @pytest.mark.asyncio
    async def test_poller_prices_with_deployment_registered_batch_rates(
        self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
    ):
        """The cost poller must price with the rates the router registered for the deployment.

        The deployment's raw model_info dict carries no litellm_params pricing, so passing
        its model_dump() made the poller bill custom-rate batches at the public cost-map
        price while the inline retrieve path billed the declared rate.
        """
        from unittest.mock import patch

        import litellm

        deployment_id = "deploy-poller-registered-rates-1"
        litellm.model_cost[deployment_id] = {
            "id": deployment_id,
            "input_cost_per_token_batches": 2e-06,
            "output_cost_per_token_batches": 4e-06,
            "litellm_provider": "bedrock",
            "mode": "chat",
        }

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
        mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        mock_job = MagicMock()
        mock_job.id = "job-poller-rates-1"
        mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
        mock_job.created_by = "user-1"
        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])

        mock_response = MagicMock()
        mock_response.status = "completed"
        mock_response.output_file_id = "file-output-123"
        mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"completed"}'

        mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
        mock_llm_router.get_deployment_credentials_with_provider = MagicMock(
            return_value={"custom_llm_provider": "bedrock", "aws_region_name": "us-east-1"}
        )

        mock_deployment = MagicMock()
        mock_deployment.litellm_params.custom_llm_provider = "bedrock"
        mock_deployment.litellm_params.model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
        mock_deployment.model_info.model_dump.return_value = {}
        mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)

        mock_file_content = MagicMock()
        mock_file_content.content = b'{"recordId":"req-1"}'

        decoded_id = f"llm_model_id,{deployment_id};llm_batch_id,batch-456;"

        try:
            with (
                patch(
                    "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
                    side_effect=[decoded_id, None],
                ),
                patch(
                    "litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
                    return_value=deployment_id,
                ),
                patch(
                    "litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
                    return_value="batch-456",
                ),
                patch(
                    "litellm.files.main.afile_content",
                    new_callable=AsyncMock,
                    return_value=mock_file_content,
                ),
                patch(
                    "litellm.batches.batch_utils.get_file_content_as_dictionary",
                    return_value=[{"recordId": "req-1"}],
                ),
                patch(
                    "litellm.batches.batch_utils.calculate_batch_cost_and_usage",
                    new_callable=AsyncMock,
                    return_value=_batch_cost_result(
                        0.0052, {"prompt_tokens": 1400, "completion_tokens": 600}, ["claude-haiku-4-5"]
                    ),
                ) as mock_calculate,
                patch(
                    "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
                    return_value=("us.anthropic.claude-haiku-4-5-20251001-v1:0", "bedrock", None, None),
                ),
                patch("litellm.litellm_core_utils.litellm_logging.Logging") as mock_logging_cls,
            ):
                mock_logging_obj = MagicMock()
                mock_logging_obj.async_success_handler = AsyncMock()
                mock_logging_cls.return_value = mock_logging_obj

                await check_batch_cost_instance.check_batch_cost()
        finally:
            litellm.model_cost.pop(deployment_id, None)

        mock_calculate.assert_awaited_once()
        passed_model_info = mock_calculate.await_args.kwargs["model_info"]
        assert passed_model_info is not None, "poller must pass the deployment's registered pricing"
        assert passed_model_info["input_cost_per_token_batches"] == 2e-06
        assert passed_model_info["output_cost_per_token_batches"] == 4e-06

    @pytest.mark.asyncio
    async def test_poller_masks_api_base_credentials_before_logging(
        self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
    ):
        """Request rows mask `key=` query credentials out of api_base before it is
        logged, but the poller skips that pre-call step, so an unmasked deployment
        api_base would land verbatim on the batch cost row: regression test for the
        poller masking the same way.
        """
        import base64
        from unittest.mock import patch

        import httpx
        import respx

        from litellm.litellm_core_utils.litellm_logging import Logging

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
        mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        mock_job = MagicMock()
        mock_job.id = "job-masked-api-base-1"
        mock_job.unified_object_id = base64.urlsafe_b64encode(
            b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
        ).decode()
        mock_job.created_by = "user-1"
        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])

        mock_response = MagicMock()
        mock_response.status = "completed"
        mock_response.output_file_id = "file-output-123"
        mock_response.error_file_id = None
        mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"completed"}'
        mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
        mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})

        mock_deployment = MagicMock()
        mock_deployment.litellm_params.custom_llm_provider = "openai"
        mock_deployment.litellm_params.model = "gpt-5.4-mini"
        mock_deployment.litellm_params.api_base = "https://gateway.example.com/v1?key=AIzaSyVERYSECRET7890"
        mock_deployment.model_info.model_dump.return_value = {}
        mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)

        output_line = json.dumps(
            {
                "custom_id": "req-1",
                "response": {
                    "status_code": 200,
                    "body": {
                        "id": "chatcmpl-1",
                        "object": "chat.completion",
                        "model": "gpt-5.4-mini",
                        "choices": [
                            {
                                "index": 0,
                                "message": {"role": "assistant", "content": "hi"},
                                "finish_reason": "stop",
                            }
                        ],
                        "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
                    },
                },
                "error": None,
            }
        )

        with (
            respx.mock(assert_all_called=True) as provider,
            patch.object(  # test-quality-ok: the poller builds Logging inline, the only seam to the row it logs
                Logging, "async_success_handler", autospec=True
            ) as success_handler,
        ):
            provider.get("https://api.openai.com/v1/files/file-output-123/content").mock(
                return_value=httpx.Response(200, content=f"{output_line}\n".encode())
            )
            await check_batch_cost_instance.check_batch_cost()

        cost_row_calls = [call for call in success_handler.await_args_list if "batch_cost" in call.kwargs]
        assert len(cost_row_calls) == 1
        logged_api_base = cost_row_calls[0].args[0].litellm_params["api_base"]
        assert logged_api_base == "https://gateway.example.com/v1?key=*****7890"
        assert "VERYSECRET" not in logged_api_base

    @pytest.mark.asyncio
    async def test_primary_path_completion_update_includes_batch_processed(
        self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
    ):
        """When batch_processed column IS present, completion update must set it to True.

        This is the symmetric counterpart to test_fallback_completion_update_omits_batch_processed
        and proves the conditional on _has_batch_processed_column governs the update data.
        """
        from unittest.mock import patch

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
        mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        mock_job = MagicMock()
        mock_job.id = "job-primary-1"
        mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
        mock_job.created_by = "user-1"

        assert check_batch_cost_instance._has_batch_processed_column is True
        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])

        mock_response: Final = LiteLLMBatch(
            id="batch-1",
            status="completed",
            output_file_id="file-output-123",
            created_at=1,
            completion_window="24h",
            endpoint="/v1/chat/completions",
            input_file_id="file-input-1",
            object="batch",
        )

        mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
        mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})

        mock_deployment = MagicMock()
        mock_deployment.litellm_params.custom_llm_provider = "openai"
        mock_deployment.litellm_params.model = "gpt-4"
        mock_deployment.model_info.model_dump.return_value = {}
        mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)

        mock_file_content = MagicMock()
        mock_file_content.content = b'{"id":"req-1"}'

        decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;"

        with (
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
                side_effect=[decoded_id, None],
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
                return_value="model-123",
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
                return_value="batch-456",
            ),
            patch(
                "litellm.files.main.afile_content",
                new_callable=AsyncMock,
                return_value=mock_file_content,
            ),
            patch(
                "litellm.batches.batch_utils.get_file_content_as_dictionary",
                return_value=[{"id": "req-1"}],
            ),
            patch(
                "litellm.batches.batch_utils.calculate_batch_cost_and_usage",
                new_callable=AsyncMock,
                return_value=_batch_cost_result(
                    0.01,
                    {"prompt_tokens": 10, "completion_tokens": 5},
                    ["gpt-4"],
                ),
            ),
            patch(
                "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
                return_value=("gpt-4", "openai", None, None),
            ),
            patch("litellm.litellm_core_utils.litellm_logging.Logging") as mock_logging_cls,
        ):
            mock_logging_obj = MagicMock()
            mock_logging_obj.async_success_handler = AsyncMock()
            mock_logging_cls.return_value = mock_logging_obj

            await check_batch_cost_instance.check_batch_cost()

        assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
            "Expected update() to be called exactly once for the completed job"
        )
        update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
        assert update_data["batch_processed"] is True, (
            "update() must include batch_processed=True when column is present"
        )
        assert update_data["status"] == "complete"

    @pytest.mark.asyncio
    async def test_completed_batch_with_no_attributable_owner_still_writes_spend_log(
        self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
    ):
        """Regression: a batch created with the master key or a team-less key has
        created_by=None and team_id=None on LiteLLM_ManagedObjectTable (the table
        never stores the raw key hash). CheckBatchCost's synthetic logging_obj for
        such a batch then carries no attributable key/user/team/end-user, and
        before the fix _should_track_cost_callback silently skipped the DB write
        with no error or warning: batch_processed still became True, but no
        LiteLLM_SpendLogs row was ever written.

        Unlike the other tests in this file, this one does NOT mock
        litellm_logging.Logging or async_success_handler -- it runs the real
        logging pipeline through to _ProxyDBLogger, which is the exact gap that
        let the original bug ship undetected.
        """
        import litellm
        from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
        mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        mock_job = MagicMock()
        mock_job.id = "job-unattributed-1"
        mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
        mock_job.created_by = None
        mock_job.team_id = None

        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])

        # A real LiteLLMBatch (not a bare MagicMock): this test runs the real
        # litellm_logging.Logging pipeline, which type-checks the result via
        # isinstance(..., LiteLLMBatch) before it will compute/attach a cost.
        from litellm.types.utils import LiteLLMBatch

        mock_response = LiteLLMBatch(
            id="batch-1",
            completion_window="24h",
            created_at=1,
            endpoint="/v1/chat/completions",
            input_file_id="file-input-123",
            object="batch",
            status="completed",
            output_file_id="file-output-123",
        )

        mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
        mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})

        mock_deployment = MagicMock()
        mock_deployment.litellm_params.custom_llm_provider = "openai"
        mock_deployment.litellm_params.model = "gpt-4"
        mock_deployment.model_info.model_dump.return_value = {}
        mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)

        mock_file_content = MagicMock()
        mock_file_content.content = b'{"id":"req-1"}'

        decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;"

        db_logger = ProxyDBLogger()
        mock_update_database = AsyncMock()

        # Unlike the other tests in this file, this one runs the real
        # litellm_logging.Logging pipeline, which calls
        # _is_base64_encoded_unified_file_id an extra time (checking result.id
        # after it's reset to job.unified_object_id). Key off the argument
        # instead of a fixed-length side_effect list so the exact call count
        # doesn't matter.
        def _fake_is_base64_encoded(file_id):
            return decoded_id if file_id == mock_job.unified_object_id else None

        with (
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
                side_effect=_fake_is_base64_encoded,
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
                return_value="model-123",
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
                return_value="batch-456",
            ),
            patch(
                "litellm.files.main.afile_content",
                new_callable=AsyncMock,
                return_value=mock_file_content,
            ),
            patch(
                "litellm.batches.batch_utils.get_file_content_as_dictionary",
                return_value=[{"id": "req-1"}],
            ),
            patch(
                "litellm.batches.batch_utils.calculate_batch_cost_and_usage",
                new_callable=AsyncMock,
                return_value=_batch_cost_result(
                    0.01,
                    {"prompt_tokens": 10, "completion_tokens": 5},
                    ["gpt-4"],
                ),
            ),
            patch(
                "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
                return_value=("gpt-4", "openai", None, None),
            ),
            patch.object(litellm, "_async_success_callback", [db_logger]),
            patch(
                "litellm.proxy.proxy_server.proxy_logging_obj",
                MagicMock(
                    db_spend_update_writer=MagicMock(update_database=mock_update_database),
                    slack_alerting_instance=MagicMock(customer_spend_alert=AsyncMock()),
                ),
            ),
            patch("litellm.proxy.proxy_server.increment_spend_counters", AsyncMock()),
            patch("litellm.proxy.proxy_server.update_cache", AsyncMock()),
        ):
            await check_batch_cost_instance.check_batch_cost()

        mock_update_database.assert_awaited_once()
        assert mock_update_database.call_args.kwargs["response_cost"] == 0.01
        assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
            "the job must still be marked processed once cost tracking succeeds"
        )

    @pytest.mark.asyncio
    async def test_cost_tracking_failure_leaves_job_unprocessed(
        self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
    ):
        """LIT-4008 regression: when fetching a completed batch's results fails
        (e.g. Anthropic rejecting a msgbatch_ id on the Files API), the job must
        NOT be marked complete/batch_processed. Pre-fix the $0 spend row was
        written and batch_processed=True made it permanent; the failure must
        instead leave the row untouched so the next poll retries, without
        aborting the poll cycle.
        """
        from unittest.mock import patch

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
        mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        mock_job = MagicMock()
        mock_job.id = "job-anthropic-1"
        mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
        mock_job.created_by = "user-1"

        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])

        mock_response = MagicMock()
        mock_response.status = "completed"
        mock_response.output_file_id = "msgbatch_01WA5hdsa2Xx8w4zyPjV1frs"

        mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
        mock_llm_router.get_deployment_credentials_with_provider = MagicMock(
            return_value={"api_key": "sk-test", "custom_llm_provider": "anthropic"}
        )

        decoded_id = "llm_model_id,model-123;llm_batch_id,msgbatch_01WA5hdsa2Xx8w4zyPjV1frs;"

        with (
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
                side_effect=[decoded_id, None],
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
                return_value="model-123",
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
                return_value="msgbatch_01WA5hdsa2Xx8w4zyPjV1frs",
            ),
            patch(
                "litellm.files.main.afile_content",
                new_callable=AsyncMock,
                side_effect=Exception("File id must have `file_` prefix."),
            ),
        ):
            await check_batch_cost_instance.check_batch_cost()

        assert mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 0, (
            "a failed cost tracking attempt must not mark the job processed"
        )

    @pytest.mark.asyncio
    @pytest.mark.parametrize("terminal_status", ["failed", "expired", "cancelled"])
    async def test_terminal_status_marks_job_processed(
        self,
        check_batch_cost_instance,
        mock_prisma_client,
        mock_llm_router,
        terminal_status,
    ):
        """When the provider reports a terminal status with nothing to bill
        (failed/cancelled, or expired with no output file), the row must be written back
        with that status and batch_processed=True so it stops being polled forever.
        """
        import base64

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
        mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        mock_job = MagicMock()
        mock_job.id = "job-terminal-1"
        mock_job.unified_object_id = base64.urlsafe_b64encode(
            b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
        ).decode()
        mock_job.created_by = "user-1"

        assert check_batch_cost_instance._has_batch_processed_column is True
        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])

        mock_response = MagicMock()
        mock_response.status = terminal_status
        mock_response.output_file_id = None
        mock_response.model_dump_json.return_value = f'{{"id":"batch-1","status":"{terminal_status}"}}'

        mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)

        await check_batch_cost_instance.check_batch_cost()

        assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
            f"Expected update() to be called exactly once for a {terminal_status} job"
        )
        update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
        assert update_data["status"] == terminal_status
        assert update_data["batch_processed"] is True, (
            "terminal-status update() must set batch_processed=True so polling stops"
        )

    @pytest.mark.asyncio
    @pytest.mark.parametrize("terminal_status", ["failed", "cancelled"])
    async def test_terminal_status_persists_managed_output_file_ids(
        self,
        check_batch_cost_instance,
        mock_prisma_client,
        mock_llm_router,
        terminal_status,
    ):
        """A cancelled/failed batch with a provider error file (and no output file) must
        be persisted with unified managed file IDs, never raw provider IDs. Raw IDs
        written here leak to every later GET /batches/{id} and GET /batches because the
        terminal row is final (batch_processed=True) and read paths only resolve, never
        mint. (Any terminal status with an output file is billed through the completed
        path instead, covered by test_terminal_status_with_output_file_is_billed.)
        """
        import base64
        import json

        from litellm.types.utils import LiteLLMBatch

        unified_batch_uid = base64.urlsafe_b64encode(
            b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
        ).decode()
        raw_error_file_id = "file-terminal-err-xyz"
        raw_input_file_id = "file-terminal-in-123"
        unified_input_file_id = base64.urlsafe_b64encode(
            b"litellm_proxy:application/octet-stream;unified_id,in-1;target_model_names,gpt-5-batch"
        ).decode()
        unified_error_file_id = base64.urlsafe_b64encode(
            f"litellm_proxy:application/octet-stream;unified_id,u-2;llm_output_file_id,{raw_error_file_id}".encode()
        ).decode()

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
        mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        input_file_row = MagicMock()
        input_file_row.unified_file_id = unified_input_file_id

        def find_managed_file(where):
            if where["flat_model_file_ids"]["has"] == raw_input_file_id:
                return input_file_row
            return None

        mock_prisma_client.db.litellm_managedfiletable.find_first = AsyncMock(side_effect=find_managed_file)

        mock_job = MagicMock()
        mock_job.id = "job-terminal-mint-1"
        mock_job.unified_object_id = unified_batch_uid
        mock_job.created_by = "user-1"
        mock_job.team_id = "team-1"

        check_batch_cost_instance._has_batch_processed_column = True
        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])

        response = LiteLLMBatch(
            id="batch-456",
            completion_window="24h",
            created_at=1,
            endpoint="/v1/chat/completions",
            input_file_id=raw_input_file_id,
            object="batch",
            status=terminal_status,
            output_file_id=None,
            error_file_id=raw_error_file_id,
        )
        mock_llm_router.aretrieve_batch = AsyncMock(return_value=response)

        from litellm.proxy.openai_files_endpoints.common_utils import ManagedBatchOutputFileWriter

        mock_hook = MagicMock(spec=ManagedBatchOutputFileWriter)
        mock_hook.get_unified_output_file_id.side_effect = [unified_error_file_id]
        mock_hook.store_batch_output_file = AsyncMock()
        check_batch_cost_instance.proxy_logging_obj.get_proxy_hook.return_value = mock_hook

        await check_batch_cost_instance.check_batch_cost()

        mock_hook.get_unified_output_file_id.assert_called_once_with(
            output_file_id=raw_error_file_id,
            model_id="model-123",
            model_name="gpt-5-batch",
        )
        mock_hook.store_batch_output_file.assert_awaited_once_with(
            unified_file_id=unified_error_file_id,
            provider_file_id=raw_error_file_id,
            model_id="model-123",
            model_name="gpt-5-batch",
            owner=mock_hook.store_batch_output_file.await_args.kwargs["owner"],
            litellm_parent_otel_span=None,
            size_bytes=None,
            fetch_provider_details=True,
        )
        owner = mock_hook.store_batch_output_file.await_args.kwargs["owner"]
        assert owner.user_id == "user-1"
        assert owner.team_id == "team-1"

        assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2
        update_call = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args
        assert update_call.kwargs["where"] == {
            "id": "job-terminal-mint-1",
            "batch_processed": False,
            "updated_at": mock_job.updated_at,
        }
        update_data = update_call.kwargs["data"]
        assert update_data["status"] == terminal_status
        assert update_data["batch_processed"] is True
        persisted = json.loads(update_data["file_object"])
        assert persisted["id"] == unified_batch_uid
        assert persisted["input_file_id"] == unified_input_file_id
        assert persisted["output_file_id"] is None
        assert persisted["error_file_id"] == unified_error_file_id
        assert raw_error_file_id not in update_data["file_object"]

    @pytest.mark.asyncio
    @pytest.mark.parametrize("completed_status", ["completed", "complete"])
    async def test_completed_without_output_file_marked_processed_without_billing(
        self,
        check_batch_cost_instance,
        mock_prisma_client,
        mock_llm_router,
        completed_status,
    ):
        """#35354 regression: a terminal completed batch whose request lines all failed
        reaches `completed` with output_file_id=None (only an error_file_id).

        Pre-fix it matched neither the completed-with-output branch nor the
        failed/expired/cancelled branch, so batch_processed stayed False and the row
        was re-selected on every poll cycle forever. It must now be marked terminal
        exactly once, without being billed: request_counts.completed == 0 proves the
        missing output file means nothing to bill rather than a lagging output id
        (#37713 keeps the lagging case eligible for the next cycle).
        """
        import base64
        from unittest.mock import patch

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
        mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        mock_job = MagicMock()
        mock_job.id = "job-completed-no-output-1"
        mock_job.unified_object_id = base64.urlsafe_b64encode(
            b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
        ).decode()
        mock_job.created_by = "user-1"

        assert check_batch_cost_instance._has_batch_processed_column is True
        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])

        mock_response = MagicMock()
        mock_response.status = completed_status
        mock_response.output_file_id = None
        mock_response.error_file_id = "file-error-123"
        mock_response.request_counts = MagicMock(completed=0, failed=3, total=3)
        mock_response.model_dump_json.return_value = f'{{"id":"batch-1","status":"{completed_status}"}}'

        mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
        # Billing reads credentials off the router; if it is touched we billed a batch
        # that has no output, which is the behaviour this test guards against.
        mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})

        with patch(
            "litellm.files.main.afile_content",
            new_callable=AsyncMock,
        ) as mock_afile_content:
            await check_batch_cost_instance.check_batch_cost()

        assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
            "a completed batch with no output file must be marked processed exactly once"
        )
        update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
        assert update_data["status"] == completed_status
        assert update_data["batch_processed"] is True, (
            "completed-without-output update() must set batch_processed=True so polling stops"
        )
        assert mock_afile_content.await_count == 0, "a batch with no output file must not be billed"
        assert mock_llm_router.get_deployment_credentials_with_provider.call_count == 0, (
            "a batch with no output file must not enter the cost-tracking path"
        )

    @pytest.mark.asyncio
    @pytest.mark.parametrize(
        "request_counts",
        [
            MagicMock(completed=7, failed=0, total=7),
            None,
            MagicMock(completed=0, failed=0, total=0),
        ],
        ids=["lagging_output_id", "unknown_counts", "synthesized_zero_counts"],
    )
    async def test_completed_with_lagging_output_file_left_for_next_cycle(
        self,
        check_batch_cost_instance,
        mock_prisma_client,
        mock_llm_router,
        request_counts,
    ):
        """#37713 regression: a batch can report completed while its output_file_id is
        still lagging behind at the provider. Retiring it in that window (or when the
        request counts cannot prove there is nothing to bill) permanently loses the
        spend record, so the poller must leave the row untouched and revisit it on the
        next cycle once the output id has appeared.
        """
        import base64
        from unittest.mock import patch

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
        mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        mock_job = MagicMock()
        mock_job.id = "job-completed-lagging-output-1"
        mock_job.unified_object_id = base64.urlsafe_b64encode(
            b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
        ).decode()
        mock_job.created_by = "user-1"

        assert check_batch_cost_instance._has_batch_processed_column is True
        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])

        mock_response = MagicMock()
        mock_response.status = "completed"
        mock_response.output_file_id = None
        mock_response.error_file_id = None
        mock_response.request_counts = request_counts

        mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
        mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})

        with patch(
            "litellm.files.main.afile_content",
            new_callable=AsyncMock,
        ) as mock_afile_content:
            await check_batch_cost_instance.check_batch_cost()

        assert mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 0, (
            "a completed batch whose output id is still lagging must stay eligible for the next poll"
        )
        assert mock_afile_content.await_count == 0, "a batch with no output file must not be billed"

    @pytest.mark.asyncio
    async def test_non_terminal_status_left_unprocessed(
        self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
    ):
        """A batch still validating/in_progress must NOT be treated as terminal: no DB
        write, so it keeps being polled until it actually reaches a terminal status.
        """
        from unittest.mock import patch

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()

        mock_job = MagicMock()
        mock_job.id = "job-in-progress-1"
        mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
        mock_job.created_by = "user-1"

        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])

        mock_response = MagicMock()
        mock_response.status = "in_progress"
        mock_response.output_file_id = None

        mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)

        decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;"

        with (
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
                side_effect=[decoded_id, None],
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
                return_value="model-123",
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
                return_value="batch-456",
            ),
        ):
            await check_batch_cost_instance.check_batch_cost()

        assert mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 0, (
            "a non-terminal batch must not be written back (would stop polling prematurely)"
        )

    @pytest.mark.asyncio
    @pytest.mark.parametrize("terminal_status", ["expired", "cancelled", "failed"])
    async def test_terminal_status_with_output_file_is_billed(
        self,
        check_batch_cost_instance,
        mock_prisma_client,
        mock_llm_router,
        terminal_status,
    ):
        """A terminal (expired/cancelled/failed) batch that still produced an output file
        served real request lines, so it must be billed (cost tracked) and then marked
        processed, not silently marked terminal without billing.
        """
        from unittest.mock import patch

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
        mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        mock_job = MagicMock()
        mock_job.id = "job-terminal-with-output-1"
        mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
        mock_job.created_by = "user-1"

        assert check_batch_cost_instance._has_batch_processed_column is True
        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])

        mock_response: Final = LiteLLMBatch(
            id="batch-1",
            status=terminal_status,
            output_file_id="file-output-123",
            created_at=1,
            completion_window="24h",
            endpoint="/v1/chat/completions",
            input_file_id="file-input-1",
            object="batch",
        )

        mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
        mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})

        mock_deployment = MagicMock()
        mock_deployment.litellm_params.custom_llm_provider = "openai"
        mock_deployment.litellm_params.model = "gpt-4"
        mock_deployment.model_info.model_dump.return_value = {}
        mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)

        mock_file_content = MagicMock()
        mock_file_content.content = b'{"id":"req-1"}'

        decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;"

        with (
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
                side_effect=[decoded_id, None],
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
                return_value="model-123",
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
                return_value="batch-456",
            ),
            patch(
                "litellm.files.main.afile_content",
                new_callable=AsyncMock,
                return_value=mock_file_content,
            ) as mock_afile_content,
            patch(
                "litellm.batches.batch_utils.get_file_content_as_dictionary",
                return_value=[{"id": "req-1"}],
            ),
            patch(
                "litellm.batches.batch_utils.calculate_batch_cost_and_usage",
                new_callable=AsyncMock,
                return_value=_batch_cost_result(
                    0.01,
                    {"prompt_tokens": 10, "completion_tokens": 5},
                    ["gpt-4"],
                ),
            ),
            patch(
                "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
                return_value=("gpt-4", "openai", None, None),
            ),
            patch("litellm.litellm_core_utils.litellm_logging.Logging") as mock_logging_cls,
        ):
            mock_logging_obj = MagicMock()
            mock_logging_obj.async_success_handler = AsyncMock()
            mock_logging_cls.return_value = mock_logging_obj

            await check_batch_cost_instance.check_batch_cost()

        assert mock_afile_content.await_count == 1, (
            f"{terminal_status} batch with an output file must fetch results and be billed"
        )
        mock_logging_obj.async_success_handler.assert_awaited_once()
        assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2
        update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
        assert update_data["batch_processed"] is True
        assert update_data["status"] == terminal_status, (
            f"billed {terminal_status} batch must keep its real terminal status in the DB"
        )

    @pytest.mark.asyncio
    async def test_error_file_failures_add_to_failed_request_count(
        self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
    ):
        """OpenAI-shaped providers report per-request failures only in a separate
        error file. The poller prices from the output file, so without also counting
        the error file's lines, batch_failed_requests on the spend log undercounts:
        regression test for the poller path merging error-file failures.
        """
        import base64
        from unittest.mock import patch

        import httpx
        import respx

        from litellm.litellm_core_utils.litellm_logging import Logging

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
        mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        mock_job = MagicMock()
        mock_job.id = "job-error-file-1"
        mock_job.unified_object_id = base64.urlsafe_b64encode(
            b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
        ).decode()
        mock_job.created_by = "user-1"
        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])

        mock_response = MagicMock()
        mock_response.status = "completed"
        mock_response.output_file_id = "file-output-123"
        mock_response.error_file_id = "file-error-456"
        mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"completed"}'
        mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
        mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})

        mock_deployment = MagicMock()
        mock_deployment.litellm_params.custom_llm_provider = "openai"
        mock_deployment.litellm_params.model = "gpt-4"
        mock_deployment.model_info.model_dump.return_value = {}
        mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)

        succeeded_line = json.dumps(
            {
                "custom_id": "req-1",
                "response": {
                    "status_code": 200,
                    "body": {
                        "id": "chatcmpl-1",
                        "object": "chat.completion",
                        "model": "gpt-4",
                        "choices": [
                            {
                                "index": 0,
                                "message": {"role": "assistant", "content": "hi"},
                                "finish_reason": "stop",
                            }
                        ],
                        "usage": {
                            "prompt_tokens": 10,
                            "completion_tokens": 5,
                            "total_tokens": 15,
                        },
                    },
                },
                "error": None,
            }
        )
        rejected_line = json.dumps(
            {
                "custom_id": "req-2",
                "response": {
                    "status_code": 400,
                    "body": {"error": {"message": "bad request"}},
                },
                "error": None,
            }
        )
        error_file_lines = "\n".join(
            json.dumps({"custom_id": custom_id, "error": {"message": "rejected"}}) for custom_id in ("req-3", "req-4")
        )

        with (
            respx.mock(assert_all_called=True) as provider,
            patch.object(  # test-quality-ok: the poller builds Logging inline, the only seam to its handler kwargs
                Logging, "async_success_handler", new_callable=AsyncMock
            ) as success_handler,
        ):
            provider.get("https://api.openai.com/v1/files/file-output-123/content").mock(
                return_value=httpx.Response(200, content=f"{succeeded_line}\n{rejected_line}\n".encode())
            )
            provider.get("https://api.openai.com/v1/files/file-error-456/content").mock(
                return_value=httpx.Response(200, content=f"{error_file_lines}\n\n".encode())
            )
            await check_batch_cost_instance.check_batch_cost()

        spend_log_calls = [call.kwargs for call in success_handler.await_args_list if "batch_cost" in call.kwargs]
        assert len(spend_log_calls) == 1
        handler_kwargs = spend_log_calls[0]
        assert handler_kwargs["batch_successful_requests"] == 1
        assert handler_kwargs["batch_failed_requests"] == 3, (
            "2 error-file lines must add to the output file's 1 rejected request"
        )
        assert handler_kwargs["batch_models"] == ["gpt-4"]
        assert handler_kwargs["batch_usage"].total_tokens == 15
        assert handler_kwargs["batch_cost"] > 0

    @pytest.mark.asyncio
    async def test_terminal_batch_with_missing_output_file_is_retired_unbilled(
        self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
    ):
        """A terminal batch whose advertised output file 404s at the provider has
        nothing to fetch on this or any later poll (Vertex AI advertises an output
        path for every batch, even ones that never wrote it), so the job must be
        retired as terminal on the first cycle instead of retrying until the
        staleness sweep gives up on it.
        """
        import base64
        from unittest.mock import patch

        from litellm.exceptions import NotFoundError

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
        mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        mock_job = MagicMock()
        mock_job.id = "job-output-gone-1"
        mock_job.unified_object_id = base64.urlsafe_b64encode(
            b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
        ).decode()
        mock_job.created_by = "user-1"

        assert check_batch_cost_instance._has_batch_processed_column is True
        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])

        missing_output_file_id = "gs://batch-out/job-1/predictions.jsonl"
        mock_response = MagicMock()
        mock_response.status = "failed"
        mock_response.output_file_id = missing_output_file_id
        mock_response.error_file_id = None
        mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"failed"}'

        mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
        mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})

        with (
            patch(
                "litellm.files.main.afile_content",
                new_callable=AsyncMock,
                side_effect=NotFoundError(
                    message=f"404: output file {missing_output_file_id} does not exist",
                    model="gemini-2.5-pro",
                    llm_provider="vertex_ai",
                ),
            ) as mock_afile_content,
            patch(
                "litellm.batches.batch_utils.calculate_batch_cost_and_usage",
                new_callable=AsyncMock,
            ) as mock_calculate,
        ):
            await check_batch_cost_instance.check_batch_cost()

        assert mock_afile_content.await_count == 1
        mock_calculate.assert_not_awaited()
        assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, (
            "a terminal batch with a 404ing output file must be retired, not retried forever"
        )
        update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
        assert update_data["status"] == "failed"
        assert update_data["batch_processed"] is True

    @pytest.mark.asyncio
    async def test_raw_output_file_id_converted_to_managed_id(
        self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
    ):
        """CheckBatchCost must convert a raw provider output_file_id to a managed base64 ID.

        Without this, GET /batches/{id} returns a raw file ID that cannot be routed
        through the proxy, causing API_KEY errors when clients call GET /files/{id}/content.
        """
        from litellm.proxy.openai_files_endpoints.common_utils import ManagedBatchOutputFileWriter

        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
        mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        mock_job = MagicMock()
        mock_job.id = "job-raw-file-1"
        mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
        mock_job.created_by = "user-1"
        mock_job.team_id = None

        check_batch_cost_instance._has_batch_processed_column = True
        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])

        raw_output_file_id = "file-batch-output-abc123"
        raw_error_file_id = "file-batch-error-xyz456"
        fake_managed_output_id = "bGl0ZWxsbV9wcm94eTo6b3V0cHV0"
        fake_managed_error_id = "bGl0ZWxsbV9wcm94eTo6ZXJyb3I="

        mock_response = MagicMock()
        mock_response.status = "completed"
        mock_response.output_file_id = raw_output_file_id
        mock_response.error_file_id = raw_error_file_id
        mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"completed"}'

        mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
        mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})

        mock_deployment = MagicMock()
        mock_deployment.litellm_params.custom_llm_provider = "azure"
        mock_deployment.litellm_params.model = "azure/gpt-5-mini"
        mock_deployment.model_name = "gpt-5-batch"
        mock_deployment.model_info.model_dump.return_value = {}
        mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)

        mock_hook = MagicMock(spec=ManagedBatchOutputFileWriter)
        mock_hook.get_unified_output_file_id.side_effect = [
            fake_managed_output_id,
            fake_managed_error_id,
        ]
        mock_hook.store_batch_output_file = AsyncMock()
        check_batch_cost_instance.proxy_logging_obj.get_proxy_hook.return_value = mock_hook

        mock_file_content = MagicMock()
        mock_file_content.content = b'{"id":"req-1"}'
        decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;"

        with (
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
                # call 1: job unified_object_id decode, call 2: existing raw check for output_file_id,
                # call 3: fix guard for output_file_id, call 4: fix guard for error_file_id
                side_effect=[decoded_id, None, None, None],
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
                return_value="model-123",
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
                return_value="batch-456",
            ),
            patch(
                "litellm.files.main.afile_content",
                new_callable=AsyncMock,
                return_value=mock_file_content,
            ),
            patch(
                "litellm.batches.batch_utils.get_file_content_as_dictionary",
                return_value=[{"id": "req-1"}],
            ),
            patch(
                "litellm.batches.batch_utils.calculate_batch_cost_and_usage",
                new_callable=AsyncMock,
                return_value=_batch_cost_result(
                    0.01,
                    {"prompt_tokens": 10, "completion_tokens": 5},
                    ["gpt-4"],
                ),
            ),
            patch(
                "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
                return_value=("gpt-5-mini", "azure", None, None),
            ),
            patch("litellm.litellm_core_utils.litellm_logging.Logging") as mock_logging_cls,
        ):
            mock_logging_obj = MagicMock()
            mock_logging_obj.async_success_handler = AsyncMock()
            mock_logging_cls.return_value = mock_logging_obj

            await check_batch_cost_instance.check_batch_cost()

        assert mock_hook.get_unified_output_file_id.call_count == 2
        mock_hook.get_unified_output_file_id.assert_any_call(
            output_file_id=raw_output_file_id,
            model_id="model-123",
            model_name="gpt-5-batch",
        )
        mock_hook.get_unified_output_file_id.assert_any_call(
            output_file_id=raw_error_file_id,
            model_id="model-123",
            model_name="gpt-5-batch",
        )
        assert mock_hook.store_batch_output_file.await_count == 2
        assert {call.kwargs["model_name"] for call in mock_hook.store_batch_output_file.await_args_list} == {
            "gpt-5-batch"
        }
        stored = {
            c.kwargs["provider_file_id"]: c.kwargs["unified_file_id"]
            for c in mock_hook.store_batch_output_file.call_args_list
        }
        assert stored == {
            raw_output_file_id: fake_managed_output_id,
            raw_error_file_id: fake_managed_error_id,
        }
        stored_sizes = {
            c.kwargs["provider_file_id"]: c.kwargs["size_bytes"]
            for c in mock_hook.store_batch_output_file.call_args_list
        }
        assert stored_sizes[raw_output_file_id] == len(mock_file_content.content)
        assert stored_sizes[raw_error_file_id] is None
        assert mock_response.output_file_id == fake_managed_output_id
        assert mock_response.error_file_id == fake_managed_error_id


class TestUnmanagedVertexRouting:
    """Routing of unmanaged Vertex batches whose unified_object_id is a raw provider job id."""

    def _instance(self, track_unmanaged, router):
        from litellm_enterprise.proxy.common_utils.check_batch_cost import (
            CheckBatchCost,
        )

        return CheckBatchCost(
            proxy_logging_obj=MagicMock(),
            prisma_client=MagicMock(),
            llm_router=router,
            track_unmanaged_batch_cost=track_unmanaged,
        )

    def _job(self, file_object=None):
        job = MagicMock()
        job.unified_object_id = "8823717160934178816"
        job.file_object = file_object if file_object is not None else _unmanaged_vertex_file_object()
        return job

    def test_flag_off_skips_unmanaged_id_unchanged(self):
        """Default (flag off): a raw numeric unified_object_id is skipped exactly as before;
        no model derivation or router lookup happens."""
        router = MagicMock()
        instance = self._instance(track_unmanaged=False, router=router)
        prom = MagicMock()

        with patch(_IS_B64, return_value=False):
            result = instance._resolve_job_routing(self._job(), prom)

        assert result is None
        prom.record_check_batch_cost_error.assert_called_once_with("invalid_unified_id")
        router.resolve_model_name_from_model_id.assert_not_called()
        router.get_model_ids.assert_not_called()

    def _vertex_deployment(self):
        deployment = MagicMock()
        deployment.litellm_params.custom_llm_provider = "vertex_ai"
        deployment.litellm_params.model = "vertex_ai/gemini-2.5-flash"
        return deployment

    def test_flag_on_routes_to_vertex_deployment(self):
        """Flag on: derive the bare model from the gs:// path, resolve it to a deployment id,
        and use the raw unified_object_id as the provider batch id."""
        router = MagicMock()
        router.resolve_model_name_from_model_id.return_value = "gemini-2.5-flash"
        router.get_model_ids.return_value = ["deploy-1"]
        router.get_deployment = MagicMock(return_value=self._vertex_deployment())
        instance = self._instance(track_unmanaged=True, router=router)

        with patch(_IS_B64, return_value=False):
            result = instance._resolve_job_routing(self._job(), MagicMock())

        assert result == ("deploy-1", "8823717160934178816")
        # bare model name (trailing GCS segment), not the full publishers/.. path
        router.resolve_model_name_from_model_id.assert_called_once_with("gemini-2.5-flash")
        router.get_model_ids.assert_called_once_with(model_name="gemini-2.5-flash")

    def test_flag_on_routes_fine_tuned_endpoint_to_vertex_deployment(self):
        """A fine-tuned Gemini batch stores `endpoints/<id>` in the gs:// path; the bare model
        (the endpoint id) must round-trip to the deployment configured as
        `vertex_ai/gemini/<id>` (LIT-6899)."""
        endpoint_id = "7768560373388541952"
        router = MagicMock()
        router.resolve_model_name_from_model_id.return_value = None
        router.get_model_list.return_value = [
            {
                "model_name": "gemini-2.5-flash-dts-usc1",
                "litellm_params": {
                    "model": f"vertex_ai/gemini/{endpoint_id}",
                    "custom_llm_provider": "vertex_ai",
                },
                "model_info": {"id": "deploy-ft"},
            },
        ]
        instance = self._instance(track_unmanaged=True, router=router)
        job = self._job(
            file_object=_unmanaged_vertex_file_object(
                input_file_id=f"gs://bucket/litellm-vertex-files/endpoints/{endpoint_id}/abc.jsonl"
            )
        )

        with patch(_IS_B64, return_value=False):
            result = instance._resolve_job_routing(job, MagicMock())

        assert result == ("deploy-ft", "8823717160934178816")

    def test_flag_on_skips_non_vertex_deployment_sharing_model_group(self):
        """Flag on, but the only deployment for the model group is a non-vertex_ai
        provider: must not be selected, even though the model group name matches."""
        router = MagicMock()
        router.resolve_model_name_from_model_id.return_value = "gemini-2.5-flash"
        router.get_model_ids.return_value = ["deploy-openai"]
        non_vertex_deployment = MagicMock()
        non_vertex_deployment.litellm_params.custom_llm_provider = "openai"
        non_vertex_deployment.litellm_params.model = "gpt-4o"
        router.get_deployment = MagicMock(return_value=non_vertex_deployment)
        instance = self._instance(track_unmanaged=True, router=router)
        prom = MagicMock()

        with patch(_IS_B64, return_value=False):
            result = instance._resolve_job_routing(self._job(), prom)

        assert result is None
        prom.record_check_batch_cost_error.assert_called_once_with("unmanaged_no_matching_deployment")

    def test_flag_on_uses_later_vertex_deployment_with_matching_suffix(self):
        router = MagicMock()
        router.resolve_model_name_from_model_id.return_value = "azure-gemini"
        router.get_model_ids.return_value = ["deploy-azure"]
        non_vertex_deployment = MagicMock()
        non_vertex_deployment.litellm_params.custom_llm_provider = "azure"
        non_vertex_deployment.litellm_params.model = "azure/gemini-2.5-flash"
        router.get_deployment = MagicMock(return_value=non_vertex_deployment)
        router.get_model_list.return_value = [
            {
                "model_name": "azure-gemini",
                "litellm_params": {
                    "model": "azure/gemini-2.5-flash",
                    "custom_llm_provider": "azure",
                },
                "model_info": {"id": "deploy-azure"},
            },
            {
                "model_name": "vertex-gemini",
                "litellm_params": {
                    "model": "vertex_ai/gemini-2.5-flash",
                    "custom_llm_provider": "vertex_ai",
                },
                "model_info": {"id": "deploy-vertex"},
            },
        ]
        instance = self._instance(track_unmanaged=True, router=router)

        with patch(_IS_B64, return_value=False):
            result = instance._resolve_job_routing(self._job(), MagicMock())

        assert result == ("deploy-vertex", "8823717160934178816")
        router.get_model_ids.assert_called_once_with(model_name="azure-gemini")

    def test_flag_on_no_matching_deployment_records_metric(self):
        """Flag on but no vertex_ai deployment for the model: skip with a distinct metric."""
        router = MagicMock()
        router.resolve_model_name_from_model_id.return_value = None
        router.get_model_ids.return_value = []
        instance = self._instance(track_unmanaged=True, router=router)
        prom = MagicMock()

        with patch(_IS_B64, return_value=False):
            result = instance._resolve_job_routing(self._job(), prom)

        assert result is None
        prom.record_check_batch_cost_error.assert_called_once_with("unmanaged_no_matching_deployment")

    def test_flag_on_non_gcs_input_is_not_unmanaged_vertex(self):
        """Flag on, but input_file_id is not a gs:// publishers path: treat as unroutable,
        do not attempt model derivation."""
        router = MagicMock()
        instance = self._instance(track_unmanaged=True, router=router)
        prom = MagicMock()
        job = self._job(file_object=_unmanaged_vertex_file_object(input_file_id="file-abc-123"))

        with patch(_IS_B64, return_value=False):
            result = instance._resolve_job_routing(job, prom)

        assert result is None
        prom.record_check_batch_cost_error.assert_called_once_with("invalid_unified_id")
        router.resolve_model_name_from_model_id.assert_not_called()

    @pytest.mark.asyncio
    async def test_end_to_end_costs_unmanaged_batch(self):
        """Flag on, completed unmanaged batch: the poller polls Vertex with the raw job id,
        computes cost, and marks batch_processed=True. Fails before this change (the row is
        skipped at the unified-id gate)."""
        router = MagicMock()
        router.resolve_model_name_from_model_id.return_value = "gemini-2.5-flash"
        router.get_model_ids.return_value = ["deploy-1"]

        mock_response: Final = LiteLLMBatch(
            id="8823717160934178816",
            status="completed",
            output_file_id="gs://bucket/out/predictions.jsonl",
            created_at=1,
            completion_window="24h",
            endpoint="/v1/chat/completions",
            input_file_id="file-input-1",
            object="batch",
        )
        router.aretrieve_batch = AsyncMock(return_value=mock_response)
        router.get_deployment_credentials_with_provider = MagicMock(
            return_value={"vertex_project": "p", "vertex_location": "us-central1"}
        )

        deployment = MagicMock()
        deployment.litellm_params.custom_llm_provider = "vertex_ai"
        deployment.litellm_params.model = "vertex_ai/gemini-2.5-flash"
        deployment.model_name = "gemini-2.5-flash"
        deployment.model_info.model_dump.return_value = {}
        router.get_deployment = MagicMock(return_value=deployment)

        instance = self._instance(track_unmanaged=True, router=router)
        instance.proxy_logging_obj.get_proxy_hook.return_value = None
        instance._has_batch_processed_column = True

        prisma = instance.prisma_client
        prisma.db = MagicMock()
        prisma.db.litellm_managedobjecttable = MagicMock()
        prisma.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        prisma.db.litellm_managedobjecttable.update = AsyncMock()
        prisma.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[self._job()])
        prisma.db.litellm_usertable = MagicMock()
        prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        mock_file_content = MagicMock()
        mock_file_content.content = b'{"id":"req-1"}'

        with (
            patch(_IS_B64, side_effect=[False, None]),
            patch(
                "litellm.files.main.afile_content",
                new_callable=AsyncMock,
                return_value=mock_file_content,
            ),
            patch(
                "litellm.batches.batch_utils.get_file_content_as_dictionary",
                return_value=[{"id": "req-1"}],
            ),
            patch(
                "litellm.batches.batch_utils.calculate_batch_cost_and_usage",
                new_callable=AsyncMock,
                return_value=_batch_cost_result(
                    0.01,
                    {"prompt_tokens": 10, "completion_tokens": 5},
                    ["gemini-2.5-flash"],
                ),
            ),
            patch(
                "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
                return_value=("gemini-2.5-flash", "vertex_ai", None, None),
            ),
            patch("litellm.litellm_core_utils.litellm_logging.Logging") as mock_logging_cls,
        ):
            mock_logging_obj = MagicMock()
            mock_logging_obj.async_success_handler = AsyncMock()
            mock_logging_cls.return_value = mock_logging_obj

            await instance.check_batch_cost()

        router.aretrieve_batch.assert_awaited_once()
        assert router.aretrieve_batch.call_args[1]["model"] == "deploy-1"
        assert router.aretrieve_batch.call_args[1]["batch_id"] == "8823717160934178816"

        mock_logging_obj.async_success_handler.assert_awaited_once()
        assert mock_logging_obj.async_success_handler.call_args[1]["batch_cost"] == 0.01

        assert prisma.db.litellm_managedobjecttable.update_many.call_count == 2
        update_data = prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
        assert update_data["batch_processed"] is True
        assert update_data["status"] == "complete"


class TestUnmanagedBedrockRouting:
    """Routing of unmanaged Bedrock batches whose unified_object_id is a raw model-invocation-job ARN."""

    _ARN = "arn:aws:bedrock:us-east-1:298249409318:model-invocation-job/1ofb47x17jua"

    def _instance(self, track_unmanaged, router):
        from litellm_enterprise.proxy.common_utils.check_batch_cost import (
            CheckBatchCost,
        )

        return CheckBatchCost(
            proxy_logging_obj=MagicMock(),
            prisma_client=MagicMock(),
            llm_router=router,
            track_unmanaged_batch_cost=track_unmanaged,
        )

    def _job(self, file_object=None):
        job = MagicMock()
        job.unified_object_id = self._ARN
        job.file_object = file_object if file_object is not None else _unmanaged_bedrock_file_object()
        return job

    def _bedrock_deployment(self):
        deployment = MagicMock()
        deployment.litellm_params.custom_llm_provider = "bedrock"
        deployment.litellm_params.model = "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
        return deployment

    def test_flag_off_skips_arn_unified_id_unchanged(self):
        """Default (flag off): a raw ARN unified_object_id is skipped exactly as before; no
        model derivation or router lookup happens."""
        router = MagicMock()
        instance = self._instance(track_unmanaged=False, router=router)
        prom = MagicMock()

        with patch(_IS_B64, return_value=False):
            result = instance._resolve_job_routing(self._job(), prom)

        assert result is None
        prom.record_check_batch_cost_error.assert_called_once_with("invalid_unified_id")
        router.resolve_model_name_from_model_id.assert_not_called()
        router.get_model_ids.assert_not_called()

    def test_flag_on_routes_to_bedrock_deployment(self):
        """Flag on: derive the bare model from the s3:// object key (":" restored to "-" is
        matched fuzzily), resolve it to a deployment id, and use the raw ARN as the batch id."""
        router = MagicMock()
        router.resolve_model_name_from_model_id.return_value = "claude-sonnet-4"
        router.get_model_ids.return_value = ["deploy-1"]
        router.get_deployment = MagicMock(return_value=self._bedrock_deployment())
        instance = self._instance(track_unmanaged=True, router=router)

        with patch(_IS_B64, return_value=False):
            result = instance._resolve_job_routing(self._job(), MagicMock())

        assert result == ("deploy-1", self._ARN)

    def test_flag_on_skips_non_bedrock_deployment_sharing_model_group(self):
        """Flag on, but the only deployment for the model group is a non-bedrock provider:
        must not be selected, even though the model group name matches."""
        router = MagicMock()
        router.resolve_model_name_from_model_id.return_value = "claude-sonnet-4"
        router.get_model_ids.return_value = ["deploy-anthropic"]
        non_bedrock_deployment = MagicMock()
        non_bedrock_deployment.litellm_params.custom_llm_provider = "anthropic"
        non_bedrock_deployment.litellm_params.model = "claude-sonnet-4-20250514"
        router.get_deployment = MagicMock(return_value=non_bedrock_deployment)
        instance = self._instance(track_unmanaged=True, router=router)
        prom = MagicMock()

        with patch(_IS_B64, return_value=False):
            result = instance._resolve_job_routing(self._job(), prom)

        assert result is None
        prom.record_check_batch_cost_error.assert_called_once_with("unmanaged_no_matching_deployment")

    def test_flag_on_matches_deployment_despite_colon_dash_mismatch(self):
        """The S3 object key has ':' replaced with '-' (e.g. 'v1-0'), but the configured
        deployment's actual bedrock model id uses ':' (e.g. 'v1:0'). Routing must still match."""
        router = MagicMock()
        router.resolve_model_name_from_model_id.return_value = None
        router.get_model_ids.return_value = []
        router.get_model_list.return_value = [
            {
                "model_name": "claude-sonnet-4",
                "litellm_params": {
                    "model": "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0",
                    "custom_llm_provider": "bedrock",
                },
                "model_info": {"id": "deploy-bedrock"},
            }
        ]
        instance = self._instance(track_unmanaged=True, router=router)

        with patch(_IS_B64, return_value=False):
            result = instance._resolve_job_routing(self._job(), MagicMock())

        assert result == ("deploy-bedrock", self._ARN)

    def test_flag_on_no_matching_deployment_records_metric(self):
        """Flag on but no bedrock deployment for the model: skip with a distinct metric."""
        router = MagicMock()
        router.resolve_model_name_from_model_id.return_value = None
        router.get_model_ids.return_value = []
        router.get_model_list.return_value = []
        instance = self._instance(track_unmanaged=True, router=router)
        prom = MagicMock()

        with patch(_IS_B64, return_value=False):
            result = instance._resolve_job_routing(self._job(), prom)

        assert result is None
        prom.record_check_batch_cost_error.assert_called_once_with("unmanaged_no_matching_deployment")

    def test_flag_on_non_s3_input_is_not_unmanaged_bedrock(self):
        """Flag on, but input_file_id is not a litellm-bedrock-files- s3:// key: treat as
        unroutable, do not attempt model derivation."""
        router = MagicMock()
        instance = self._instance(track_unmanaged=True, router=router)
        prom = MagicMock()
        job = self._job(file_object=_unmanaged_bedrock_file_object(input_file_id="file-abc-123"))

        with patch(_IS_B64, return_value=False):
            result = instance._resolve_job_routing(job, prom)

        assert result is None
        prom.record_check_batch_cost_error.assert_called_once_with("invalid_unified_id")
        router.resolve_model_name_from_model_id.assert_not_called()

    @pytest.mark.asyncio
    async def test_end_to_end_costs_unmanaged_batch(self):
        """Flag on, completed unmanaged batch: the poller polls Bedrock with the raw ARN,
        computes cost, and marks batch_processed=True."""
        router = MagicMock()
        router.resolve_model_name_from_model_id.return_value = "claude-sonnet-4"
        router.get_model_ids.return_value = ["deploy-1"]

        mock_response: Final = LiteLLMBatch(
            id=self._ARN,
            status="completed",
            output_file_id="s3://bucket/out/predictions.jsonl",
            created_at=1,
            completion_window="24h",
            endpoint="/v1/chat/completions",
            input_file_id="file-input-1",
            object="batch",
        )
        router.aretrieve_batch = AsyncMock(return_value=mock_response)
        router.get_deployment_credentials_with_provider = MagicMock(return_value={"aws_region_name": "us-east-1"})

        deployment = self._bedrock_deployment()
        deployment.model_name = "claude-sonnet-4"
        deployment.model_info.model_dump.return_value = {}
        router.get_deployment = MagicMock(return_value=deployment)

        instance = self._instance(track_unmanaged=True, router=router)
        instance.proxy_logging_obj.get_proxy_hook.return_value = None
        instance._has_batch_processed_column = True

        prisma = instance.prisma_client
        prisma.db = MagicMock()
        prisma.db.litellm_managedobjecttable = MagicMock()
        prisma.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
        prisma.db.litellm_managedobjecttable.update = AsyncMock()
        prisma.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[self._job()])
        prisma.db.litellm_usertable = MagicMock()
        prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        mock_file_content = MagicMock()
        mock_file_content.content = b'{"id":"req-1"}'

        with (
            patch(_IS_B64, side_effect=[False, None]),
            patch(
                "litellm.files.main.afile_content",
                new_callable=AsyncMock,
                return_value=mock_file_content,
            ),
            patch(
                "litellm.batches.batch_utils.get_file_content_as_dictionary",
                return_value=[{"id": "req-1"}],
            ),
            patch(
                "litellm.batches.batch_utils.calculate_batch_cost_and_usage",
                new_callable=AsyncMock,
                return_value=_batch_cost_result(
                    0.02,
                    {"prompt_tokens": 10, "completion_tokens": 5},
                    ["claude-sonnet-4"],
                ),
            ),
            patch(
                "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
                return_value=("claude-sonnet-4", "bedrock", None, None),
            ),
            patch("litellm.litellm_core_utils.litellm_logging.Logging") as mock_logging_cls,
        ):
            mock_logging_obj = MagicMock()
            mock_logging_obj.async_success_handler = AsyncMock()
            mock_logging_cls.return_value = mock_logging_obj

            await instance.check_batch_cost()

        router.aretrieve_batch.assert_awaited_once()
        assert router.aretrieve_batch.call_args[1]["model"] == "deploy-1"
        assert router.aretrieve_batch.call_args[1]["batch_id"] == self._ARN

        mock_logging_obj.async_success_handler.assert_awaited_once()
        assert mock_logging_obj.async_success_handler.call_args[1]["batch_cost"] == 0.02

        assert prisma.db.litellm_managedobjecttable.update_many.call_count == 2
        update_data = prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"]
        assert update_data["batch_processed"] is True
        assert update_data["status"] == "complete"


class TestUnmanagedBatchCostFlagIsGeneralized:
    """The single track_unmanaged_batch_cost flag must cover both Vertex and Bedrock, not
    just the provider it was originally added for."""

    def test_one_flag_routes_both_vertex_and_bedrock_jobs(self):
        from litellm_enterprise.proxy.common_utils.check_batch_cost import (
            CheckBatchCost,
        )

        router = MagicMock()
        router.resolve_model_name_from_model_id.side_effect = [
            "gemini-2.5-flash",
            "claude-sonnet-4",
        ]
        router.get_model_ids.side_effect = [["deploy-vertex"], ["deploy-bedrock"]]

        def _get_deployment(model_id):
            if model_id == "deploy-vertex":
                deployment = MagicMock()
                deployment.litellm_params.custom_llm_provider = "vertex_ai"
                deployment.litellm_params.model = "vertex_ai/gemini-2.5-flash"
                return deployment
            deployment = MagicMock()
            deployment.litellm_params.custom_llm_provider = "bedrock"
            deployment.litellm_params.model = "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
            return deployment

        router.get_deployment = MagicMock(side_effect=_get_deployment)

        instance = CheckBatchCost(
            proxy_logging_obj=MagicMock(),
            prisma_client=MagicMock(),
            llm_router=router,
            track_unmanaged_batch_cost=True,
        )

        vertex_job = MagicMock()
        vertex_job.unified_object_id = "8823717160934178816"
        vertex_job.file_object = _unmanaged_vertex_file_object()

        bedrock_job = MagicMock()
        bedrock_job.unified_object_id = TestUnmanagedBedrockRouting._ARN
        bedrock_job.file_object = _unmanaged_bedrock_file_object()

        with patch(_IS_B64, return_value=False):
            vertex_result = instance._resolve_job_routing(vertex_job, MagicMock())
            bedrock_result = instance._resolve_job_routing(bedrock_job, MagicMock())

        assert vertex_result == ("deploy-vertex", "8823717160934178816")
        assert bedrock_result == ("deploy-bedrock", TestUnmanagedBedrockRouting._ARN)


class TestManagedOutputFileIdEncodesPublicModelGroup:
    """LIT-4964 regression: the unified output file id created by the background poller must
    encode the public model group as ``target_model_names``, not the provider model.

    Key model-access checks resolve a managed file id back to a model via ``target_model_names``,
    so encoding the provider model (e.g. ``gpt-5.5``) makes
    ``GET /v1/files/{output_file_id}/content`` fail for every key.
    """

    _PUBLIC_MODEL_GROUP = "gpt-5-batch"
    _RAW_OUTPUT_FILE_ID = "file-batch-output-abc123"

    @staticmethod
    def _managed_input_file_id(model_group: str) -> str:
        import base64

        unified_id = (
            "litellm_proxy:application/octet-stream;unified_id,c4843482-b176-4901-8292-7523fd0f2c6e;"
            f"target_model_names,{model_group}"
        )
        return base64.urlsafe_b64encode(unified_id.encode()).decode().rstrip("=")

    def _job(self, input_file_id: str) -> MagicMock:
        from litellm.types.utils import LiteLLMBatch

        job = MagicMock()
        job.id = "job-lit-4964"
        job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
        job.created_by = "user-1"
        job.team_id = None
        job.file_object = LiteLLMBatch(
            id="batch-456",
            completion_window="24h",
            created_at=1,
            endpoint="/v1/chat/completions",
            input_file_id=input_file_id,
            object="batch",
            status="completed",
        ).model_dump_json()
        return job

    async def _run(self, job: MagicMock) -> str:
        from litellm_enterprise.proxy.common_utils.check_batch_cost import (
            CheckBatchCost,
        )

        from enterprise.litellm_enterprise.proxy.hooks.managed_files import PROXY_LiteLLMManagedFiles
        from litellm.proxy.openai_files_endpoints.common_utils import ManagedBatchOutputFileWriter
        from litellm.types.utils import LiteLLMBatch

        router = MagicMock()
        router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})
        deployment = MagicMock()
        deployment.litellm_params.custom_llm_provider = "azure"
        deployment.litellm_params.model = "azure/gpt-5.5"
        deployment.model_name = self._PUBLIC_MODEL_GROUP
        deployment.model_info.model_dump.return_value = {}
        router.get_deployment = MagicMock(return_value=deployment)

        hook = MagicMock(spec=ManagedBatchOutputFileWriter)
        hook.get_unified_output_file_id = lambda output_file_id, model_id, model_name: (
            PROXY_LiteLLMManagedFiles.get_unified_output_file_id(
                None, output_file_id=output_file_id, model_id=model_id, model_name=model_name
            )
        )
        hook.store_batch_output_file = AsyncMock()
        proxy_logging_obj = MagicMock()
        proxy_logging_obj.get_proxy_hook.return_value = hook

        prisma_client = MagicMock()
        prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)

        instance = CheckBatchCost(
            proxy_logging_obj=proxy_logging_obj,
            prisma_client=prisma_client,
            llm_router=router,
        )

        response = LiteLLMBatch(
            id="batch-456",
            completion_window="24h",
            created_at=1,
            endpoint="/v1/chat/completions",
            input_file_id=job.file_object,
            object="batch",
            status="completed",
        )
        response.output_file_id = self._RAW_OUTPUT_FILE_ID

        file_content = MagicMock()
        file_content.content = b'{"id":"req-1"}'

        with (
            patch(
                "litellm.files.main.afile_content",
                new_callable=AsyncMock,
                return_value=file_content,
            ),
            patch(
                "litellm.batches.batch_utils.get_file_content_as_dictionary",
                return_value=[{"id": "req-1"}],
            ),
            patch(
                "litellm.batches.batch_utils.calculate_batch_cost_and_usage",
                new_callable=AsyncMock,
                return_value=_batch_cost_result(0.01, {"prompt_tokens": 10}, ["gpt-5.5"]),
            ),
            patch("litellm.litellm_core_utils.litellm_logging.Logging") as logging_cls,
        ):
            logging_obj = MagicMock()
            logging_obj.async_success_handler = AsyncMock()
            logging_cls.return_value = logging_obj

            await instance._track_completed_batch_cost(
                job=job,
                response=response,
                model_id="model-123",
                batch_id="batch-456",
                prom_logger=None,
            )

        return response.output_file_id

    @pytest.mark.asyncio
    async def test_target_model_names_comes_from_input_file_not_provider_model(self):
        from litellm.proxy.openai_files_endpoints.common_utils import (
            is_base64_encoded_unified_file_id,
            get_models_from_unified_file_id,
        )

        output_file_id = await self._run(self._job(self._managed_input_file_id(self._PUBLIC_MODEL_GROUP)))

        decoded = is_base64_encoded_unified_file_id(output_file_id)
        assert get_models_from_unified_file_id(decoded) == [self._PUBLIC_MODEL_GROUP]
        assert "gpt-5.5" not in decoded

    @pytest.mark.asyncio
    async def test_key_scoped_to_model_group_can_read_the_output_file(self):
        from litellm.proxy._types import UserAPIKeyAuth
        from litellm.proxy.auth.auth_checks import can_key_call_model
        from litellm.proxy.auth.auth_utils import (
            _extract_models_from_managed_resource_id,
        )

        output_file_id = await self._run(self._job(self._managed_input_file_id(self._PUBLIC_MODEL_GROUP)))

        models = _extract_models_from_managed_resource_id(output_file_id, "file_id", None)
        assert models == [self._PUBLIC_MODEL_GROUP]
        assert (
            await can_key_call_model(
                model=models[0],
                llm_model_list=None,
                valid_token=UserAPIKeyAuth(api_key="sk-test", models=[self._PUBLIC_MODEL_GROUP]),
                llm_router=None,
            )
            is True
        )

    @pytest.mark.asyncio
    async def test_falls_back_to_deployment_model_group_without_managed_input_file(self):
        from litellm.proxy.openai_files_endpoints.common_utils import (
            is_base64_encoded_unified_file_id,
            get_models_from_unified_file_id,
        )

        output_file_id = await self._run(self._job("file-raw-provider-input"))

        decoded = is_base64_encoded_unified_file_id(output_file_id)
        assert get_models_from_unified_file_id(decoded) == [self._PUBLIC_MODEL_GROUP]


class TestBatchCostAttribution:
    """CheckBatchCost rebuilds the creator's spend metadata from the managed-object row so
    the batch-cost log is attributed like a non-batch request."""

    def _prisma(self, key_row=None, team_row=None, user_row=None):
        prisma = MagicMock()
        prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=key_row)
        prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=team_row)
        prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=user_row)
        return prisma

    def _instance(self, key_row=None, team_row=None, user_row=None, prisma=None):
        from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
            ManagedObjectCreatorAttribution,
        )

        return ManagedObjectCreatorAttribution(
            prisma_client=prisma if prisma is not None else self._prisma(key_row, team_row, user_row),
            poller_name="CheckBatchCost",
        )

    def _job(self, **overrides):
        from types import SimpleNamespace

        fields = {
            "created_by": "alice",
            "team_id": "team-alpha",
            "api_key": "hash-alice",
            "request_tags": ["env:prod"],
        }
        fields.update(overrides)
        return SimpleNamespace(unified_object_id="uoi", **fields)

    @pytest.mark.asyncio
    async def test_metadata_carries_key_team_and_tags(self):
        """The spend row names the creating key, its team, both aliases, and the tags."""
        from types import SimpleNamespace

        instance = self._instance(
            key_row=SimpleNamespace(key_alias="prod-key"),
            team_row=SimpleNamespace(team_alias="Team Alpha"),
            user_row=SimpleNamespace(user_email="[email protected]", user_alias=None),
        )

        metadata = await instance.batch_metadata(self._job(), "batch-1")

        assert metadata["user_api_key"] == "hash-alice"
        assert metadata["user_api_key_hash"] == "hash-alice"
        assert metadata["user_api_key_user_id"] == "alice"
        assert metadata["user_api_key_team_id"] == "team-alpha"
        assert metadata["user_api_key_alias"] == "prod-key"
        assert metadata["user_api_key_team_alias"] == "Team Alpha"
        assert metadata["tags"] == ["env:prod"]

    @pytest.mark.asyncio
    async def test_metadata_tolerates_legacy_row_without_columns(self):
        """Rows created before the columns existed carry only created_by/team_id and must
        still produce an attributed row rather than raising."""
        instance = self._instance()
        job = self._job(api_key=None, request_tags=None)

        metadata = await instance.batch_metadata(job, "batch-1")

        assert metadata["user_api_key"] is None
        assert metadata["user_api_key_user_id"] == "alice"
        assert metadata["user_api_key_team_id"] == "team-alpha"
        assert "tags" not in metadata

    @pytest.mark.asyncio
    async def test_metadata_keeps_key_when_team_key_has_no_user(self):
        """A team-scoped key carries no user id. The user lookup is skipped (prisma rejects
        a None user_id) and the key hash still drives key-level attribution."""
        from types import SimpleNamespace

        prisma = self._prisma(key_row=SimpleNamespace(key_alias="svc-key"))
        instance = self._instance(prisma=prisma)
        job = self._job(created_by=None)

        metadata = await instance.batch_metadata(job, "batch-1")

        assert metadata["user_api_key"] == "hash-alice"
        assert metadata["user_api_key_user_id"] is None
        assert metadata["user_api_key_alias"] == "svc-key"
        prisma.db.litellm_usertable.find_unique.assert_not_called()

    @pytest.mark.asyncio
    async def test_batch_metadata_keeps_the_creator_column_and_adds_no_budgets(self):
        """The batch cost event keeps the user id and fields the batch poll has always written,
        even though background responses read budgets and drop key-scoped creators off the same rows."""
        from types import SimpleNamespace

        instance = self._instance(
            key_row=SimpleNamespace(key_alias="svc-key", spend=4.0, max_budget=5.0),
            team_row=SimpleNamespace(team_alias="Team Alpha", organization_id=None, spend=40.0, max_budget=50.0),
        )

        metadata = await instance.batch_metadata(self._job(created_by="key:hash-alice"), "batch-1")

        assert metadata["user_api_key_user_id"] == "key:hash-alice"
        assert [field for field in metadata if field.endswith(("_spend", "_max_budget"))] == []

    @pytest.mark.asyncio
    async def test_metadata_drops_non_string_tags(self):
        """Non-string tags are dropped so a malformed stored value cannot slip past the
        tag-budget checks that consume this metadata."""
        instance = self._instance()
        job = self._job(request_tags=["env:prod", 7, None, "team:ml"])

        metadata = await instance.batch_metadata(job, "batch-1")

        assert metadata["tags"] == ["env:prod", "team:ml"]

    @pytest.mark.asyncio
    async def test_key_alias_lookup_failure_does_not_break_attribution(self):
        """An alias lookup failure must not lose the spend row; the key hash and team still
        attribute it."""
        prisma = self._prisma()
        prisma.db.litellm_verificationtoken.find_unique = AsyncMock(side_effect=Exception("db down"))
        instance = self._instance(prisma=prisma)

        metadata = await instance.batch_metadata(self._job(), "batch-1")

        assert metadata["user_api_key"] == "hash-alice"
        assert metadata.get("user_api_key_alias") is None

    @pytest.mark.asyncio
    async def test_cli_session_batch_keeps_its_alias_without_a_key_row(self):
        instance = self._instance(key_row=None)

        metadata = await instance.batch_metadata(self._job(api_key="cli-session-alice"), "batch-1")

        assert metadata["user_api_key"] == "cli-session-alice"
        assert metadata["user_api_key_alias"] == "cli-session-alice"

    @pytest.mark.asyncio
    async def test_raw_cli_session_token_on_a_legacy_batch_row_is_not_treated_as_the_alias(self):
        instance = self._instance(key_row=None)

        metadata = await instance.batch_metadata(self._job(api_key="cli-session-Qm7xJ2kP9sLw4vT1nR8yAa"), "batch-1")

        assert metadata.get("user_api_key_alias") is None

    @pytest.mark.asyncio
    async def test_unnamed_key_keeps_the_creating_user_alias(self):
        """Regression: a key generated without key_alias resolves to no alias, and the
        overwrite must not null out the creating user's alias that _get_user_info supplied.
        Most keys carry no alias, so this is the common batch, not an edge case."""
        from types import SimpleNamespace

        instance = self._instance(
            key_row=SimpleNamespace(key_alias=None),
            user_row=SimpleNamespace(user_email="[email protected]", user_alias="Alice Chen"),
        )

        metadata = await instance.batch_metadata(self._job(), "batch-1")

        assert metadata["user_api_key_alias"] == "Alice Chen"
        assert metadata["user_api_key"] == "hash-alice"

    @pytest.mark.asyncio
    async def test_rotated_key_keeps_the_creating_user_alias(self):
        """Batches outlive keys. When the creating key has been rotated or deleted the
        lookup returns no row, and the spend log keeps a resolvable name instead of null."""
        from types import SimpleNamespace

        instance = self._instance(
            key_row=None,
            user_row=SimpleNamespace(user_email="[email protected]", user_alias="Alice Chen"),
        )

        metadata = await instance.batch_metadata(self._job(), "batch-1")

        assert metadata["user_api_key_alias"] == "Alice Chen"

    @pytest.mark.asyncio
    async def test_named_key_still_owns_the_alias(self):
        """The fallback must not weaken the intended precedence: a key that has its own
        alias still overrides the creating user's."""
        from types import SimpleNamespace

        instance = self._instance(
            key_row=SimpleNamespace(key_alias="prod-key"),
            user_row=SimpleNamespace(user_email="[email protected]", user_alias="Alice Chen"),
        )

        metadata = await instance.batch_metadata(self._job(), "batch-1")

        assert metadata["user_api_key_alias"] == "prod-key"

    @pytest.mark.asyncio
    async def test_org_id_snapshotted_on_the_row_wins(self):
        """The org_id column captures the creating key's organization at submission time,
        like team_id, so a key later moved to another org still bills the original one."""
        from types import SimpleNamespace

        instance = self._instance(
            key_row=SimpleNamespace(key_alias="prod-key", organization_id="org-moved-to"),
            team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"),
        )

        metadata = await instance.batch_metadata(self._job(org_id="org-at-creation"), "batch-1")

        assert metadata["user_api_key_org_id"] == "org-at-creation"

    @pytest.mark.asyncio
    async def test_org_id_comes_from_the_creating_key(self):
        """The spend update writer increments organization spend from user_api_key_org_id.
        A legacy row without the org_id column falls back to the creating key's org."""
        from types import SimpleNamespace

        instance = self._instance(
            key_row=SimpleNamespace(key_alias="prod-key", organization_id="org-42"),
            team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"),
        )

        metadata = await instance.batch_metadata(self._job(), "batch-1")

        assert metadata["user_api_key_org_id"] == "org-42"

    @pytest.mark.asyncio
    async def test_org_id_falls_back_to_the_team_organization(self):
        """A key with no org of its own still books batch spend against its team's
        organization, matching how the request path resolves org attribution."""
        from types import SimpleNamespace

        instance = self._instance(
            key_row=SimpleNamespace(key_alias="prod-key", organization_id=None),
            team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"),
        )

        metadata = await instance.batch_metadata(self._job(), "batch-1")

        assert metadata["user_api_key_org_id"] == "org-team"

    @pytest.mark.asyncio
    async def test_key_lookup_failure_still_bills_the_team_org(self):
        """A key-table error while resolving a legacy row's org must not drop the team's
        organization: the two lookups fail independently, so org spend still lands."""
        from types import SimpleNamespace

        prisma = self._prisma(team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"))
        prisma.db.litellm_verificationtoken.find_unique = AsyncMock(side_effect=Exception("db down"))
        instance = self._instance(prisma=prisma)

        metadata = await instance.batch_metadata(self._job(), "batch-1")

        assert metadata["user_api_key_org_id"] == "org-team"

    @pytest.mark.asyncio
    async def test_no_org_leaves_the_key_unset(self):
        """Without any org the key is absent entirely, so the spend writer's org update
        stays skipped instead of matching an empty-string organization."""
        from types import SimpleNamespace

        instance = self._instance(
            key_row=SimpleNamespace(key_alias="prod-key", organization_id=None),
            team_row=SimpleNamespace(team_alias="Team Alpha", organization_id=None),
        )

        metadata = await instance.batch_metadata(self._job(), "batch-1")

        assert "user_api_key_org_id" not in metadata

    @pytest.mark.asyncio
    async def test_metadata_provenance_keeps_spend_log_api_key_joinable(self):
        """
        CheckBatchCost stores the VerificationToken hash on the managed object. The
        spend-log writer must receive matching user_api_key_hash provenance so it
        does not re-hash that value; otherwise DailyUserSpend.api_key no longer joins
        VerificationToken and Usage shows key-hash-... with a null alias/email.
        """
        from datetime import datetime, timezone
        from types import SimpleNamespace

        from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
        from litellm.proxy.utils import hash_token

        token_hash = hash_token("sk-batch-creator-key")
        instance = self._instance(
            key_row=SimpleNamespace(key_alias="prod-key"),
            user_row=SimpleNamespace(user_email="[email protected]", user_alias=None),
        )
        metadata = await instance.batch_metadata(self._job(api_key=token_hash), "batch-1")

        assert metadata["user_api_key"] == token_hash
        assert metadata["user_api_key_hash"] == token_hash

        payload = get_logging_payload(
            kwargs={
                "model": "gpt-4o",
                "call_type": "aretrieve_batch",
                "litellm_params": {"metadata": metadata},
            },
            response_obj={
                "id": "batch_123",
                "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
            },
            start_time=datetime.now(timezone.utc),
            end_time=datetime.now(timezone.utc),
        )
        assert payload["api_key"] == token_hash
        assert payload["api_key"] != hash_token(token_hash)


class TestPollPageStarvation:
    """LIT-5462 regression: a row that can never be costed used to keep its slot in the
    MAX_OBJECTS_PER_POLL_CYCLE page forever, so once enough of them accumulated no newer
    batch was ever polled or costed."""

    def _instance(self, prisma, llm_router):
        from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost

        proxy_logging_obj = MagicMock()
        proxy_logging_obj.get_proxy_hook.return_value = None
        return CheckBatchCost(
            proxy_logging_obj=proxy_logging_obj,
            prisma_client=prisma,
            llm_router=llm_router,
        )

    def _prisma(self, jobs):
        prisma = MagicMock()
        prisma.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=0)
        prisma.db.litellm_managedobjecttable.update = AsyncMock()
        prisma.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=jobs)
        prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
        return prisma

    def _job(self, job_id, unified_object_id):
        job = MagicMock()
        job.id = job_id
        job.unified_object_id = unified_object_id
        job.created_by = "user-1"
        return job

    @staticmethod
    def _encode(unified_id: str) -> str:
        import base64

        return base64.urlsafe_b64encode(unified_id.encode()).decode().rstrip("=")

    @pytest.mark.asyncio
    async def test_unified_id_without_model_id_is_retired(self):
        """A unified id that decodes but carries no model_id is unroutable no matter what
        the config says, so it must leave the poll page instead of being retried forever."""
        prisma = self._prisma([self._job("job-no-model", self._encode("litellm_proxy;llm_batch_id:poison-no-model"))])
        llm_router = MagicMock()
        llm_router.aretrieve_batch = AsyncMock()

        await self._instance(prisma, llm_router).check_batch_cost()

        llm_router.aretrieve_batch.assert_not_awaited()
        prisma.db.litellm_managedobjecttable.update_many.assert_awaited()
        call = prisma.db.litellm_managedobjecttable.update_many.call_args[1]
        assert call["where"]["id"] == "job-no-model"
        assert call["where"]["batch_processed"] is False
        assert call["data"] == {"batch_processed": True}

    @pytest.mark.asyncio
    async def test_provider_404_retires_job(self):
        """The provider dropping its record of the batch is permanent: no later retrieve
        can succeed, so the row must stop occupying a slot."""
        import litellm

        prisma = self._prisma(
            [
                self._job(
                    "job-gone",
                    self._encode("litellm_proxy;model_id:model-123;llm_batch_id:batch_deadbeef"),
                )
            ]
        )
        llm_router = MagicMock()
        llm_router.aretrieve_batch = AsyncMock(
            side_effect=litellm.NotFoundError(
                message="No batch found with id 'batch_deadbeef'.",
                model="model-123",
                llm_provider="openai",
            )
        )

        await self._instance(prisma, llm_router).check_batch_cost()

        prisma.db.litellm_managedobjecttable.update_many.assert_awaited()
        assert prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"] == {"batch_processed": True}

    @pytest.mark.asyncio
    async def test_provider_404_with_deployment_gone_keeps_job(self):
        """With the batch's own deployment removed from the router, default fallbacks can
        send the retrieve to a provider that never saw the batch. That 404 proves nothing,
        so the row must stay unprocessed instead of losing its spend forever."""
        import litellm

        prisma = self._prisma(
            [
                self._job(
                    "job-misrouted",
                    self._encode("litellm_proxy;model_id:model-gone;llm_batch_id:batch_alive"),
                )
            ]
        )
        llm_router = MagicMock()
        llm_router.get_deployment = MagicMock(return_value=None)
        llm_router.aretrieve_batch = AsyncMock(
            side_effect=litellm.NotFoundError(
                message="No batch found with id 'batch_alive'.",
                model="model-gone",
                llm_provider="openai",
            )
        )

        await self._instance(prisma, llm_router).check_batch_cost()

        prisma.db.litellm_managedobjecttable.update.assert_not_awaited()

    @pytest.mark.asyncio
    async def test_transient_provider_error_keeps_job_for_retry(self):
        """A failure that may clear up (timeout, 5xx) must still leave the row unprocessed."""
        prisma = self._prisma(
            [
                self._job(
                    "job-flaky",
                    self._encode("litellm_proxy;model_id:model-123;llm_batch_id:batch_flaky"),
                )
            ]
        )
        llm_router = MagicMock()
        llm_router.aretrieve_batch = AsyncMock(side_effect=Exception("connection reset"))

        await self._instance(prisma, llm_router).check_batch_cost()

        prisma.db.litellm_managedobjecttable.update.assert_not_awaited()

    @pytest.mark.asyncio
    async def test_retirement_falls_back_to_status_without_batch_processed_column(self):
        """Older schemas have no batch_processed column, so the only way to stop selecting
        the row is the status filter the poll query already applies."""
        prisma = self._prisma([self._job("job-legacy", self._encode("litellm_proxy;llm_batch_id:poison-no-model"))])
        instance = self._instance(prisma, MagicMock())
        instance._has_batch_processed_column = False

        await instance.check_batch_cost()

        assert prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"] == {"status": "stale_expired"}

    @pytest.mark.asyncio
    async def test_newer_batch_is_polled_once_dead_rows_are_retired(self):
        """The end state the customer cares about: dead rows retire on the cycle they are
        first seen, and the healthy batch behind them keeps getting polled."""
        dead_rows = [
            self._job("job-no-model", self._encode("litellm_proxy;llm_batch_id:poison-no-model")),
            self._job(
                "job-gone",
                self._encode("litellm_proxy;model_id:model-123;llm_batch_id:batch_deadbeef"),
            ),
        ]
        live_row = self._job(
            "job-live",
            self._encode("litellm_proxy;model_id:model-123;llm_batch_id:batch_live"),
        )
        prisma = self._prisma(dead_rows + [live_row])

        import litellm

        in_progress = MagicMock()
        in_progress.status = "in_progress"

        async def _retrieve(model, batch_id, litellm_metadata):
            if batch_id == "batch_deadbeef":
                raise litellm.NotFoundError(
                    message=f"No batch found with id '{batch_id}'.",
                    model=model,
                    llm_provider="openai",
                )
            return in_progress

        llm_router = MagicMock()
        llm_router.aretrieve_batch = AsyncMock(side_effect=_retrieve)

        await self._instance(prisma, llm_router).check_batch_cost()

        retired = [
            call[1]["where"]["id"]
            for call in prisma.db.litellm_managedobjecttable.update_many.call_args_list
            if "id" in call[1]["where"]
        ]
        assert retired == ["job-no-model", "job-gone"]
        assert llm_router.aretrieve_batch.await_args_list[-1][1]["batch_id"] == "batch_live", (
            "the newer healthy batch must still be polled in the same cycle"
        )

    @pytest.mark.asyncio
    async def test_404_that_does_not_name_the_batch_keeps_job_for_retry(self):
        """A 404 about something other than the batch, e.g. a renamed Azure deployment, is
        fixable in config, so the row must survive to be costed after the fix."""
        import litellm

        prisma = self._prisma(
            [
                self._job(
                    "job-bad-deployment",
                    self._encode("litellm_proxy;model_id:model-123;llm_batch_id:batch_real"),
                )
            ]
        )
        llm_router = MagicMock()
        llm_router.aretrieve_batch = AsyncMock(
            side_effect=litellm.NotFoundError(
                message="Error code: 404 - DeploymentNotFound",
                model="model-123",
                llm_provider="azure",
            )
        )

        await self._instance(prisma, llm_router).check_batch_cost()

        prisma.db.litellm_managedobjecttable.update.assert_not_awaited()


class _FakeManagedObjectRow:
    """One managed batch row the provider has finished but nothing has costed yet."""

    def __init__(self):
        self.id = "job-claim-1"
        self.unified_object_id = _CLAIM_UNIFIED_BATCH_ID
        self.model_object_id = "batch-456"
        self.file_purpose = "batch"
        self.status = "in_progress"
        self.batch_processed = False
        self.created_by = "user-1"
        self.team_id = None
        self.api_key = None
        self.request_tags = None
        self.created_at = datetime(9999, 1, 1, tzinfo=timezone.utc)
        self.updated_at = datetime(2000, 1, 1, tzinfo=timezone.utc)
        self.file_object = json.dumps(
            {
                "id": "batch-456",
                "status": "in_progress",
                "input_file_id": "file-input-1",
                "output_file_id": _CLAIM_OUTPUT_FILE_ID,
            }
        )


class _FakeManagedObjectTable:
    """A LiteLLM_ManagedObjectTable double backed by one real, mutable row.

    It honours the batch_processed and status filters, so the poller's compare-and-swap
    and the managed-files deletion guard both read the same state a shared Postgres row
    would give them. Staleness sweeps (the only queries scoped by created_at) never match.
    """

    def __init__(self, row: _FakeManagedObjectRow, journal: list):
        self.row = row
        self.journal = journal
        self.update_many = AsyncMock(side_effect=self._update_many)
        self.update = AsyncMock(side_effect=self._update)
        self.find_many = AsyncMock(side_effect=self._find_many)
        self.find_first = AsyncMock(return_value=None)

    def _matches(self, where: dict) -> bool:
        for key, value in where.items():
            if key == "created_at":
                if "lt" in value and not self.row.created_at < value["lt"]:
                    return False
                continue
            if key == "status":
                if self.row.status in value.get("not_in", []):
                    return False
                if "in" in value and self.row.status not in value["in"]:
                    return False
            elif getattr(self.row, key) != value:
                return False
        return True

    async def _update_many(self, *, where: dict, data: dict) -> int:
        if not self._matches(where):
            return 0
        if "updated_at" in where:
            self.journal.append("finalize")
        elif "batch_processed" in where:
            self.journal.append("claim" if data.get("batch_processed") else "release")
        for key, value in data.items():
            setattr(self.row, key, value)
        return 1

    async def _update(self, *, where: dict, data: dict) -> None:
        self.journal.append("finalize")
        for key, value in data.items():
            setattr(self.row, key, value)

    async def _find_many(self, *, where: dict, take=None, order=None) -> list:
        return [self.row] if self._matches(where) else []


class TestMultiPodBatchCostClaim:
    """LIT-4827 regression: every pod and uvicorn worker schedules its own poller against
    the shared LiteLLM_ManagedObjectTable, so a completed batch must be claimed atomically
    before its cost is logged. Without the claim two pods select the same row in one window
    and both write an aretrieve_batch spend log for it, double counting the spend.

    The claim sits immediately before the spend-log write rather than before the results
    fetch, because batch_processed is also what keeps an unbilled row selectable by later
    poll cycles and what blocks deletion of the files the fetch reads."""

    @staticmethod
    def _instance(prisma, llm_router):
        from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost

        proxy_logging_obj = MagicMock()
        proxy_logging_obj.get_proxy_hook.return_value = None
        return CheckBatchCost(
            proxy_logging_obj=proxy_logging_obj,
            prisma_client=prisma,
            llm_router=llm_router,
        )

    @staticmethod
    def _prisma(row: _FakeManagedObjectRow, journal: list):
        prisma = MagicMock()
        prisma.db.litellm_managedobjecttable = _FakeManagedObjectTable(row, journal)
        prisma.db.litellm_managedfiletable.find_many = AsyncMock(return_value=[])
        prisma.db.litellm_managedfiletable.find_first = AsyncMock(return_value=None)
        prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
        prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None)
        return prisma

    @staticmethod
    def _router():
        from litellm.types.utils import LiteLLMBatch

        response: Final = LiteLLMBatch(
            id="batch-456",
            status="completed",
            output_file_id=_CLAIM_OUTPUT_FILE_ID,
            error_file_id=None,
            created_at=1,
            completed_at=2,
            completion_window="24h",
            endpoint="/v1/chat/completions",
            input_file_id="file-input-1",
            object="batch",
        )

        deployment = MagicMock()
        deployment.litellm_params.custom_llm_provider = "openai"
        deployment.litellm_params.model = "gpt-4"
        deployment.model_info.model_dump.return_value = {}

        router = MagicMock()
        router.aretrieve_batch = AsyncMock(return_value=response)
        router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})
        router.get_deployment = MagicMock(return_value=deployment)
        return router

    @staticmethod
    @contextmanager
    def _billing_patches(journal: list, during_fetch=None, bill_error=None):
        """Patch the cost path a batch runs through, journalling the results fetch and the
        spend-log write. during_fetch runs while the output file is being read, which is
        the window an interrupted worker or a concurrent file deletion lands in."""
        file_content = MagicMock()
        file_content.content = b'{"id":"req-1"}'

        async def _afile_content(**kwargs):
            journal.append("fetch")
            if during_fetch is not None:
                await during_fetch()
            return file_content

        async def _bill(**kwargs):
            journal.append("bill")
            if bill_error is not None:
                raise bill_error

        def _is_b64(file_id):
            if file_id == _CLAIM_UNIFIED_BATCH_ID:
                return "llm_model_id,model-123;llm_batch_id,batch-456;"
            return False

        logging_obj = MagicMock()
        logging_obj.async_success_handler = AsyncMock(side_effect=_bill)

        with (
            patch(_IS_B64, side_effect=_is_b64),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
                return_value="model-123",
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
                return_value="batch-456",
            ),
            patch("litellm.files.main.afile_content", new=AsyncMock(side_effect=_afile_content)),
            patch(
                "litellm.batches.batch_utils.get_file_content_as_dictionary",
                return_value=[{"id": "req-1"}],
            ),
            patch(
                "litellm.batches.batch_utils.calculate_batch_cost_and_usage",
                new_callable=AsyncMock,
                return_value=_batch_cost_result(0.01, {"prompt_tokens": 10, "completion_tokens": 5}, ["gpt-4"]),
            ),
            patch(
                "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
                return_value=("gpt-4", "openai", None, None),
            ),
            patch("litellm.litellm_core_utils.litellm_logging.Logging", return_value=logging_obj),
        ):
            yield logging_obj

    @pytest.mark.asyncio
    @pytest.mark.parametrize("failure_phase", ("before_write", "after_write", "external"))
    @pytest.mark.parametrize("collector_enabled", (False, True))
    async def test_builtin_accounting_retries_only_before_a_possible_charge(
        self, failure_phase: str, collector_enabled: bool
    ) -> None:
        import base64
        from datetime import datetime

        import httpx
        import litellm
        import respx

        from litellm.caching.caching import DualCache
        from litellm.integrations.custom_logger import CustomLogger
        from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
        from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
        from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
        from litellm.proxy.utils import PrismaClient
        from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
        from litellm.types.utils import SpecialEnums

        class AccountingDependency:
            def __init__(self) -> None:
                self.failure_phase = failure_phase
                self.writer: Final = DBSpendUpdateWriter()
                self.failed_tracking_alert = AsyncMock()
                self.slack_alerting_instance = MagicMock()
                self.slack_alerting_instance.customer_spend_alert = AsyncMock(
                    side_effect=ConnectionError("Alert destination unavailable after charge")
                    if failure_phase == "after_write"
                    else None
                )

            @property
            def db_spend_update_writer(self) -> DBSpendUpdateWriter:
                if self.failure_phase == "before_write":
                    raise ConnectionError("Accounting dependency temporarily unavailable")
                return self.writer

        class ExternalCallback(CustomLogger):
            async def async_log_success_event(
                self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
            ) -> None:
                if failure_phase == "external":
                    raise ConnectionError("External destination unavailable")

        budget_cache: Final = DualCache()
        RouterBudgetLimiting(
            dual_cache=budget_cache,
            provider_budget_config={"openai": {"budget_limit": 100, "time_period": "1d"}},
            model_list=[
                {
                    "model_name": "batch-alias",
                    "litellm_params": {"model": "openai/batch-test-model", "max_budget": 100, "budget_duration": "1d"},
                    "model_info": {"id": "model-123"},
                }
            ],
        )
        dependency: Final = AccountingDependency()
        spend_prisma: Final = PrismaClient(
            database_url="postgresql://unused:unused@localhost:5432/unused", proxy_logging_obj=dependency
        )
        create_spend_rows: Final = AsyncMock(return_value=1)
        input_rate: Final = 0.000002
        litellm.register_model(
            {
                "batch-test-model": {
                    "litellm_provider": "openai",
                    "mode": "chat",
                    "input_cost_per_token_batches": input_rate,
                    "output_cost_per_token_batches": 0.000006,
                }
            }
        )
        content: Final = json.dumps(
            {
                "response": {
                    "status_code": 200,
                    "body": {
                        "model": "batch-test-model",
                        "usage": {"prompt_tokens": 100, "completion_tokens": 0, "total_tokens": 100},
                    },
                },
                "error": None,
            }
        ).encode()
        row: Final = _FakeManagedObjectRow()
        original_status: Final = row.status
        original_file_object: Final = row.file_object
        row.unified_object_id = (
            base64.urlsafe_b64encode(
                SpecialEnums.LITELLM_MANAGED_BATCH_COMPLETE_STR.value.format("model-123", "batch-456").encode()
            )
            .decode()
            .rstrip("=")
        )
        prisma: Final = self._prisma(row, [])
        router: Final = self._router()
        router.get_deployment.return_value = Deployment(
            model_name="batch-alias",
            litellm_params=LiteLLM_Params(model="openai/batch-test-model"),
            model_info=ModelInfo(id="model-123"),
        )
        from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer

        producer: Final = MagicMock(spec=SpendEventProducer) if collector_enabled else None
        litellm.logging_callback_manager.add_litellm_async_success_callback(
            ProxyDBLogger(spend_event_producer=producer)
        )
        litellm.logging_callback_manager.add_litellm_async_success_callback(ExternalCallback())
        with (
            respx.mock,
            patch("litellm.proxy.proxy_server.proxy_logging_obj", dependency),
            patch("litellm.proxy.proxy_server.prisma_client", spend_prisma),
            patch.object(
                spend_prisma.writer_db._original_prisma,
                "litellm_spendlogs",
                MagicMock(create_many=create_spend_rows),
            ),
        ):
            respx.get(f"https://api.openai.com/v1/files/{_CLAIM_OUTPUT_FILE_ID}/content").mock(
                return_value=httpx.Response(200, content=content)
            )
            worker: Final = self._instance(prisma, router)
            await worker.check_batch_cost()
            assert row.batch_processed is (failure_phase != "before_write")
            if failure_phase == "before_write":
                assert row.status == original_status
                assert json.loads(row.file_object) == original_file_object
            assert create_spend_rows.await_count == (0 if failure_phase == "before_write" else 1)
            dependency.failure_phase = "recovered"
            await worker.check_batch_cost()
            await worker.check_batch_cost()

        assert row.batch_processed is True
        assert await budget_cache.async_get_cache("provider_spend:openai:1d") == pytest.approx(100 * input_rate)
        assert await budget_cache.async_get_cache("deployment_spend:model-123:1d") == pytest.approx(100 * input_rate)
        create_spend_rows.assert_awaited_once()
        assert create_spend_rows.call_args.kwargs["data"][0]["spend"] == pytest.approx(100 * input_rate)
        if producer is not None:
            assert all(
                json.loads(call.args[0])["call_type"] != "aretrieve_batch" for call in producer.publish.call_args_list
            )

    @pytest.mark.asyncio
    @pytest.mark.parametrize(
        ("deployment_model", "successes", "failures", "expected_model"),
        (
            ("openai/", 0, 2, "openai/"),
            ("", 0, 2, "model-123"),
            ("openai/", 2, 1, "batch-test-model"),
            ("openai/batch-test-model", 2, 1, "batch-test-model"),
            ("openai/batch-test-model", 2, 0, "batch-test-model"),
        ),
    )
    @pytest.mark.parametrize("failure_envelope", (None, 1, "invalid", [1]))
    async def test_batch_accounting_preserves_models_and_mixed_results(
        self, deployment_model: str, successes: int, failures: int, expected_model: str, failure_envelope: object
    ) -> None:
        import base64
        from datetime import datetime

        import httpx
        import litellm
        import respx

        from litellm.integrations.custom_logger import CustomLogger
        from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
        from litellm.types.utils import LiteLLMBatch, SpecialEnums

        input_rate: Final = 0.000002
        output_rate: Final = 0.000006
        litellm.register_model(
            {
                "batch-test-model": {
                    "litellm_provider": "openai",
                    "mode": "chat",
                    "input_cost_per_token_batches": input_rate,
                    "output_cost_per_token_batches": output_rate,
                }
            }
        )
        success: Final = {
            "response": {
                "status_code": 200,
                "body": {
                    "model": "batch-test-model",
                    "usage": {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120},
                },
            },
            "error": None,
        }
        failure: Final = {"response": failure_envelope, "error": {"code": "batch_expired", "message": "expired"}}
        rows: Final = (success,) * successes + (failure,) * failures
        content: Final = b"\n".join(json.dumps(row).encode() for row in rows)

        class BatchSpendRecorder(CustomLogger):
            def __init__(self) -> None:
                super().__init__()
                self.calls: tuple[dict[str, object], ...] = ()

            async def async_log_success_event(
                self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
            ) -> None:
                if kwargs["call_type"] == "aretrieve_batch":
                    self.calls += (kwargs,)

        callback: Final = BatchSpendRecorder()
        litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
        row: Final = _FakeManagedObjectRow()
        row.unified_object_id = (
            base64.urlsafe_b64encode(
                SpecialEnums.LITELLM_MANAGED_BATCH_COMPLETE_STR.value.format("model-123", "batch-456").encode()
            )
            .decode()
            .rstrip("=")
        )
        prisma: Final = self._prisma(row, [])
        router: Final = self._router()
        router.get_deployment.return_value = Deployment(
            model_name="batch-alias",
            litellm_params=LiteLLM_Params(model=deployment_model, custom_llm_provider="openai"),
            model_info=ModelInfo(id="model-123"),
        )
        router.aretrieve_batch.return_value = LiteLLMBatch(
            id="batch-456",
            completion_window="24h",
            created_at=1,
            endpoint="/v1/chat/completions",
            input_file_id="file-input-1",
            output_file_id=_CLAIM_OUTPUT_FILE_ID,
            object="batch",
            status="completed",
        )
        router.get_deployment_credentials_with_provider.return_value = {
            "api_key": "synthetic-key",
            "custom_llm_provider": "openai",
        }
        with respx.mock:
            respx.get(f"https://api.openai.com/v1/files/{_CLAIM_OUTPUT_FILE_ID}/content").mock(
                return_value=httpx.Response(200, content=content)
            )
            instance: Final = self._instance(prisma, router)
            await instance.check_batch_cost()
            await instance.check_batch_cost()

        assert row.batch_processed is True
        assert row.status == "complete"
        router.aretrieve_batch.assert_awaited_once()
        assert len(callback.calls) == 1
        payload: Final = callback.calls[0]["standard_logging_object"]
        assert payload["model"] == expected_model
        assert payload["response_cost"] == pytest.approx(successes * (100 * input_rate + 20 * output_rate))
        assert payload["prompt_tokens"] == successes * 100
        assert payload["completion_tokens"] == successes * 20
        assert payload["hidden_params"]["batch_successful_requests"] == successes
        assert payload["hidden_params"]["batch_failed_requests"] == failures
        assert payload["hidden_params"]["batch_models"] == (
            ["batch-test-model"] if deployment_model == "openai/batch-test-model" else ["batch-test-model"] * successes
        )

    @pytest.mark.asyncio
    @pytest.mark.parametrize("error_source", ("batch", "output"))
    async def test_a_404_for_another_identifier_preserves_recovery(self, error_source: str) -> None:
        import base64

        import httpx
        import litellm
        import respx

        from litellm.integrations.custom_logger import CustomLogger
        from litellm.types.utils import SpecialEnums

        class SpendRecorder(CustomLogger):
            def __init__(self) -> None:
                super().__init__()
                self.costs: tuple[float, ...] = ()

            async def async_log_success_event(
                self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
            ) -> None:
                if kwargs["call_type"] == "aretrieve_batch":
                    cost: Final = kwargs["response_cost"]
                    assert isinstance(cost, float)
                    self.costs += (cost,)

        recorder: Final = SpendRecorder()
        litellm.logging_callback_manager.add_litellm_async_success_callback(recorder)
        litellm.register_model(
            {
                "batch-recovery-model": {
                    "litellm_provider": "openai",
                    "mode": "chat",
                    "input_cost_per_token_batches": 0.01,
                    "output_cost_per_token_batches": 0.02,
                }
            }
        )
        row: Final = _FakeManagedObjectRow()
        row.unified_object_id = (
            base64.urlsafe_b64encode(
                SpecialEnums.LITELLM_MANAGED_BATCH_COMPLETE_STR.value.format("model-123", "batch-456").encode()
            )
            .decode()
            .rstrip("=")
        )
        router: Final = self._router()
        response: Final = router.aretrieve_batch.return_value
        error: Final = litellm.NotFoundError(
            message=f"No object found with id '{'batch-456' if error_source == 'batch' else _CLAIM_OUTPUT_FILE_ID}0'.",
            model="gpt-4",
            llm_provider="openai",
        )
        router.aretrieve_batch.side_effect = (error, response) if error_source == "batch" else (response, response)
        prisma: Final = self._prisma(row, [])
        output: Final = json.dumps(
            {
                "response": {
                    "status_code": 200,
                    "body": {
                        "model": "batch-recovery-model",
                        "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
                    },
                },
                "error": None,
            }
        ).encode()
        with respx.mock:
            route: Final = respx.get(f"https://api.openai.com/v1/files/{_CLAIM_OUTPUT_FILE_ID}/content")
            route.mock(
                side_effect=error if error_source == "output" else None,
                return_value=None if error_source == "output" else httpx.Response(200, content=output),
            )
            await self._instance(prisma, router).check_batch_cost()
            assert row.batch_processed is False
            assert recorder.costs == ()
            route.mock(return_value=httpx.Response(200, content=output))
            await self._instance(prisma, router).check_batch_cost()
            await self._instance(prisma, router).check_batch_cost()

        assert row.batch_processed is True
        assert row.status == "complete"
        assert recorder.costs == (pytest.approx(10 * 0.01 + 5 * 0.02),)

    @staticmethod
    def _claim_calls(prisma) -> list:
        return [
            call.kwargs
            for call in prisma.db.litellm_managedobjecttable.update_many.call_args_list
            if "id" in call.kwargs["where"]
        ]

    @staticmethod
    async def _run_deletion_guard(prisma, file_id: str) -> None:
        """Run the real managed-files deletion guard against the row the poller is costing."""
        from litellm_enterprise.proxy.hooks.managed_files import PROXY_LiteLLMManagedFiles

        cache = MagicMock()
        cache.async_get_cache = AsyncMock(return_value=None)
        cache.async_set_cache = AsyncMock()
        guard = PROXY_LiteLLMManagedFiles(internal_usage_cache=cache, prisma_client=prisma)

        scheduler = MagicMock()
        scheduler.get_job.return_value = MagicMock()
        with patch("litellm.proxy.proxy_server.scheduler", scheduler):
            await guard._check_file_deletion_allowed(file_id)

    @pytest.mark.asyncio
    @pytest.mark.parametrize("status", ("completed", "failed", "expired", "cancelled"))
    @pytest.mark.parametrize("metadata_text", ("before\x00after", r"\u0000"))
    async def test_cost_claim_persists_terminal_metadata_without_a_later_write(
        self, status: str, metadata_text: str
    ) -> None:
        row: Final = _FakeManagedObjectRow()
        journal: Final = []
        prisma: Final = self._prisma(row, journal)
        router: Final = self._router()
        router.aretrieve_batch.return_value = router.aretrieve_batch.return_value.model_copy(
            update={"status": status, "metadata": {"annotation": metadata_text}}
        )
        prisma.db.litellm_managedobjecttable.update.side_effect = RuntimeError("final write unavailable")
        worker: Final = self._instance(prisma, router)
        with self._billing_patches(journal) as logging_obj:
            await worker.check_batch_cost()
            await worker.check_batch_cost()
        assert row.batch_processed is True
        assert row.status == ("complete" if status == "completed" else status)
        assert json.loads(row.file_object)["status"] == status
        assert json.loads(row.file_object)["metadata"] == {"annotation": metadata_text.replace("\x00", "")}
        assert logging_obj.async_success_handler.await_count == 1

    @pytest.mark.asyncio
    async def test_winning_pod_claims_the_row_between_fetching_and_billing(self):
        """The claim flips batch_processed false -> true after the results are in hand and
        before the spend log is written, so a concurrent pod's claim finds no matching row."""
        row = _FakeManagedObjectRow()
        journal = []
        prisma = self._prisma(row, journal)

        with self._billing_patches(journal) as logging_obj:
            await self._instance(prisma, self._router()).check_batch_cost()

        assert journal == ["fetch", "claim", "bill"]
        assert row.status == "complete"
        assert json.loads(row.file_object)["usage"] == {"prompt_tokens": 10, "completion_tokens": 5}
        logging_obj.async_success_handler.assert_awaited_once()
        assert row.batch_processed is True

    @pytest.mark.asyncio
    async def test_a_pod_that_loses_the_claim_after_fetching_does_not_bill(self):
        """Both pods select the row and fetch its results in the same window. The one whose
        compare-and-swap finds the row already taken must not write a second spend log."""
        row = _FakeManagedObjectRow()
        journal = []
        prisma = self._prisma(row, journal)

        async def _other_pod_wins_the_row():
            row.batch_processed = True

        with self._billing_patches(journal, during_fetch=_other_pod_wins_the_row) as logging_obj:
            await self._instance(prisma, self._router()).check_batch_cost()

        assert journal == ["fetch"]
        logging_obj.async_success_handler.assert_not_awaited()
        assert row.status == "in_progress"
        assert json.loads(row.file_object)["status"] == "in_progress"

    @pytest.mark.asyncio
    @pytest.mark.parametrize("stored_as_json_string", (True, False))
    async def test_an_unclassified_logging_failure_keeps_the_claim(self, stored_as_json_string: bool) -> None:
        row: Final = _FakeManagedObjectRow()
        row.file_object = row.file_object if stored_as_json_string else json.loads(row.file_object)
        journal: Final = []
        prisma: Final = self._prisma(row, journal)

        with self._billing_patches(journal, bill_error=Exception("Accounting outcome unknown")):
            await self._instance(prisma, self._router()).check_batch_cost()

        assert journal == ["fetch", "claim", "bill"]
        assert row.batch_processed is True

        with self._billing_patches(journal) as logging_obj:
            await self._instance(prisma, self._router()).check_batch_cost()
            await self._instance(prisma, self._router()).check_batch_cost()

        logging_obj.async_success_handler.assert_not_awaited()

    @pytest.mark.asyncio
    async def test_a_worker_interrupted_mid_costing_leaves_the_batch_billable(self):
        """A pod killed while reading a batch's results must leave the row for a later
        cycle. Claiming before the fetch marked the batch processed for good, so the pod
        that died took that batch's spend with it and no other pod ever selected it."""
        row = _FakeManagedObjectRow()
        journal = []
        prisma = self._prisma(row, journal)
        reached_fetch = asyncio.Event()

        async def _never_returns():
            reached_fetch.set()
            await asyncio.Event().wait()

        with self._billing_patches(journal, during_fetch=_never_returns) as logging_obj:
            interrupted = asyncio.create_task(self._instance(prisma, self._router()).check_batch_cost())
            await asyncio.wait_for(reached_fetch.wait(), timeout=5)
            assert row.batch_processed is False, "an in-flight costing must not mark the row processed"
            interrupted.cancel()
            with pytest.raises(asyncio.CancelledError):
                await interrupted

        assert journal == ["fetch"]
        logging_obj.async_success_handler.assert_not_awaited()

        survivor_journal = []
        survivor_prisma = self._prisma(row, survivor_journal)
        with self._billing_patches(survivor_journal) as survivor_logging:
            await self._instance(survivor_prisma, self._router()).check_batch_cost()

        assert survivor_journal == ["fetch", "claim", "bill"]
        survivor_logging.async_success_handler.assert_awaited_once()
        assert row.batch_processed is True

    @pytest.mark.asyncio
    async def test_costing_in_flight_keeps_the_referenced_file_undeletable(self):
        """The deletion guard only holds files whose batch still has batch_processed false,
        so claiming the row before the fetch let a concurrent delete remove the very output
        file the in-flight costing was about to read."""
        row = _FakeManagedObjectRow()
        journal = []
        prisma = self._prisma(row, journal)
        reached_fetch = asyncio.Event()
        finish_fetch = asyncio.Event()

        async def _wait_for_the_delete_attempt():
            reached_fetch.set()
            await finish_fetch.wait()

        with self._billing_patches(journal, during_fetch=_wait_for_the_delete_attempt):
            costing = asyncio.create_task(self._instance(prisma, self._router()).check_batch_cost())
            await asyncio.wait_for(reached_fetch.wait(), timeout=5)

            with pytest.raises(HTTPException) as blocked:
                await self._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID)
            assert blocked.value.status_code == 400
            assert _CLAIM_OUTPUT_FILE_ID in blocked.value.detail

            finish_fetch.set()
            await asyncio.wait_for(costing, timeout=5)

        assert journal == ["fetch", "claim", "bill"]
        assert row.batch_processed is True
        await self._run_deletion_guard(prisma, _CLAIM_OUTPUT_FILE_ID)

    @pytest.mark.asyncio
    async def test_schema_without_batch_processed_still_bills(self):
        """Older schemas have no column to claim, so they keep the pre-fix behavior instead
        of losing every batch's cost."""
        row = _FakeManagedObjectRow()
        journal = []
        prisma = self._prisma(row, journal)
        instance = self._instance(prisma, self._router())
        instance._has_batch_processed_column = False

        with self._billing_patches(journal) as logging_obj:
            await instance.check_batch_cost()

        assert self._claim_calls(prisma) == []
        assert journal == ["fetch", "bill", "finalize"]
        logging_obj.async_success_handler.assert_awaited_once()


class TestBatchLineItemsFromThePoller:
    """When batch_line_item_callbacks is set, the poller hands every line of a finished batch to those loggers
    once the batch is billed and retired, in the background, so a slow, hung or failing sink never unbills, re-polls
    or holds up a batch."""

    _INPUT = b'{"custom_id":"ok-1","body":{"messages":[{"role":"user","content":"hi"}]}}\n'
    _OUTPUT = (
        b'{"custom_id":"ok-1","response":{"status_code":200,"body":{"model":"gpt-4",'
        b'"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}}}\n'
    )
    _ERROR = b'{"custom_id":"ok-1","response":{"status_code":400,"body":{"error":{"message":"expired"}}}}\n'

    @staticmethod
    def _recording_logger(journal: list, error: Exception | None = None):
        from litellm.integrations.custom_logger import CustomLogger

        class _Recorder(CustomLogger):
            def __init__(self):
                super().__init__()
                self.payloads = []

            async def async_log_batch_line_items(self, payload):
                journal.append("lines")
                self.payloads.append(payload)
                if error is not None:
                    raise error

        return _Recorder()

    @staticmethod
    def _setup(journal: list, *, status="completed", output_file_id=_CLAIM_OUTPUT_FILE_ID, error_file_id=None):
        row = _FakeManagedObjectRow()
        prisma = TestMultiPodBatchCostClaim._prisma(row, journal)
        router = TestMultiPodBatchCostClaim._router()
        response = router.aretrieve_batch.return_value
        response.status = status
        response.output_file_id = output_file_id
        response.error_file_id = error_file_id
        response.input_file_id = "file-input-1"
        return row, TestMultiPodBatchCostClaim._instance(prisma, router)

    @staticmethod
    async def _stored(instance) -> None:
        """Wait for the line items the poll handed to the background."""
        await asyncio.gather(*instance._line_item_stores)

    @classmethod
    @contextmanager
    def _provider(cls, journal: list):
        """The provider's batch files by id, and a spend-log writer that journals each write."""
        files = {"file-input-1": cls._INPUT, _CLAIM_OUTPUT_FILE_ID: cls._OUTPUT, "file-err-1": cls._ERROR}

        async def _afile_content(**kwargs):
            journal.append(f"read {kwargs['file_id']}")
            if kwargs["file_id"] not in files:
                from litellm.exceptions import NotFoundError

                raise NotFoundError(
                    message=f"No file found with id '{kwargs['file_id']}'.", model="gpt-4", llm_provider="openai"
                )
            content = MagicMock()
            content.content = files[kwargs["file_id"]]
            return content

        def _is_b64(file_id):
            if file_id == _CLAIM_UNIFIED_BATCH_ID:
                return "llm_model_id,model-123;llm_batch_id,batch-456;"
            return False

        logging_obj = MagicMock()
        logging_obj.async_success_handler = AsyncMock(side_effect=lambda **kwargs: journal.append("bill"))
        with (
            patch(_IS_B64, side_effect=_is_b64),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
                return_value="model-123",
            ),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
                return_value="batch-456",
            ),
            patch("litellm.files.main.afile_content", new=AsyncMock(side_effect=_afile_content)),
            patch("litellm.litellm_core_utils.litellm_logging.Logging", return_value=logging_obj),
        ):
            yield

    @pytest.mark.asyncio
    async def test_lines_reach_the_callbacks_after_the_claimed_batch_is_billed_and_retired(self, monkeypatch):
        import litellm

        journal: list = []
        row, instance = self._setup(journal)
        logger = self._recording_logger(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))

        with self._provider(journal):
            await instance.check_batch_cost()
            await self._stored(instance)

        assert journal == [
            f"read {_CLAIM_OUTPUT_FILE_ID}",
            "claim",
            "bill",
            f"read {_CLAIM_OUTPUT_FILE_ID}",
            "read file-input-1",
            "lines",
        ]
        (payload,) = logger.payloads
        assert payload["batch_id"] == _CLAIM_UNIFIED_BATCH_ID
        assert payload["finished_at"] == 2
        assert payload["part"] == 0
        (line,) = payload["lines"]
        assert (line["custom_id"], line["status"]) == ("ok-1", "success")
        assert line["usage"] == {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}
        assert line["metadata"]["user_api_key_user_id"] == "user-1"
        assert row.batch_processed is True

    @pytest.mark.asyncio
    async def test_the_lines_are_priced_off_the_event_loop(self, monkeypatch):
        import threading

        import litellm
        from litellm.batches import batch_line_items

        journal: list = []
        row, instance = self._setup(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (self._recording_logger(journal),))
        price = batch_line_items._line_item
        threads = []

        def _price(*args):
            threads.append(threading.current_thread())
            return price(*args)

        monkeypatch.setattr(batch_line_items, "_line_item", _price)
        with self._provider(journal):
            await instance.check_batch_cost()
            await self._stored(instance)

        assert journal[-1] == "lines"
        (thread,) = threads
        assert thread is not threading.main_thread()

    @pytest.mark.asyncio
    async def test_a_creator_that_cannot_be_looked_up_gets_lines_without_bodies(self, monkeypatch):
        import litellm

        journal: list = []
        row, instance = self._setup(journal)
        row.api_key = sha256(b"sk-deleted-key").hexdigest()
        logger = self._recording_logger(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))

        with self._provider(journal):
            await instance.check_batch_cost()
            await self._stored(instance)

        (payload,) = logger.payloads
        assert [(line["request"], line["response"]) for line in payload["lines"]] == [(None, None)]
        assert "read file-input-1" not in journal

    @pytest.mark.asyncio
    async def test_a_creator_with_logging_on_gets_the_request_and_response(self, monkeypatch):
        import litellm

        journal: list = []
        row, instance = self._setup(journal)
        row.api_key = "cli-session-user-1"
        logger = self._recording_logger(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
        monkeypatch.setattr(litellm, "turn_off_message_logging", False)

        with self._provider(journal):
            await instance.check_batch_cost()
            await self._stored(instance)

        assert journal[-4:] == ["bill", f"read {_CLAIM_OUTPUT_FILE_ID}", "read file-input-1", "lines"]
        ((line,),) = [payload["lines"] for payload in logger.payloads]
        assert line["request"] == json.loads(self._INPUT)
        assert line["response"] == json.loads(self._OUTPUT)

    @pytest.mark.asyncio
    async def test_a_failing_sink_leaves_the_batch_billed_and_retired(self, monkeypatch):
        import litellm

        journal: list = []
        row, instance = self._setup(journal)
        monkeypatch.setattr(
            litellm, "batch_line_item_callbacks", (self._recording_logger(journal, RuntimeError("bucket gone")),)
        )

        with self._provider(journal):
            await instance.check_batch_cost()
            await self._stored(instance)

        assert [step for step in journal if not step.startswith("read")] == ["claim", "bill", "lines"]
        assert row.batch_processed is True
        assert row.status == "complete"

    @pytest.mark.asyncio
    async def test_without_callbacks_only_the_output_file_is_read(self, monkeypatch):
        import litellm

        journal: list = []
        _, instance = self._setup(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", ())

        with self._provider(journal):
            await instance.check_batch_cost()
            await self._stored(instance)

        assert journal == [f"read {_CLAIM_OUTPUT_FILE_ID}", "claim", "bill"]

    @pytest.mark.asyncio
    async def test_a_batch_where_every_line_failed_sends_its_error_lines_after_it_is_retired(self, monkeypatch):
        import litellm

        journal: list = []
        row, instance = self._setup(journal, status="expired", output_file_id=None, error_file_id="file-err-1")
        logger = self._recording_logger(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))

        with self._provider(journal):
            await instance.check_batch_cost()
            await self._stored(instance)

        assert journal == ["finalize", "read file-err-1", "read file-input-1", "lines"]
        (payload,) = logger.payloads
        assert [(line["custom_id"], line["status"], line["response_cost"]) for line in payload["lines"]] == [
            ("ok-1", "failure", 0.0)
        ]
        assert row.batch_processed is True

    @pytest.mark.asyncio
    async def test_a_batch_retired_with_nothing_billable_reads_its_input_file_by_the_id_the_provider_reported(
        self, monkeypatch
    ):
        import litellm

        journal: list = []
        _, instance = self._setup(journal, status="expired", output_file_id=None, error_file_id="file-err-1")
        logger = self._recording_logger(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))

        async def _persist_managed_ids(*, response, **_):
            response.input_file_id = "managed-id-of-the-copy-on-another-deployment"

        with (
            self._provider(journal),
            patch(
                "litellm.proxy.openai_files_endpoints.common_utils.ensure_batch_response_managed_file_ids",
                new=AsyncMock(side_effect=_persist_managed_ids),
            ),
        ):
            await instance.check_batch_cost()
            await self._stored(instance)

        assert journal == ["finalize", "read file-err-1", "read file-input-1", "lines"]
        ((line,),) = [payload["lines"] for payload in logger.payloads]
        assert line["request"] == json.loads(self._INPUT)

    @pytest.mark.asyncio
    async def test_a_batch_whose_files_hold_no_readable_line_sends_no_lines(self, monkeypatch):
        import litellm

        journal: list = []
        _, instance = self._setup(journal, status="expired", output_file_id=None, error_file_id="file-err-1")
        logger = self._recording_logger(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
        monkeypatch.setattr(type(self), "_ERROR", b"not json\n")

        with self._provider(journal):
            await instance.check_batch_cost()
            await self._stored(instance)

        assert journal == ["finalize", "read file-err-1", "read file-input-1"]
        assert logger.payloads == []

    @pytest.mark.asyncio
    @pytest.mark.parametrize("database_error", [False, True])
    async def test_a_batch_that_could_not_be_retired_sends_no_lines_until_it_is(
        self, monkeypatch: pytest.MonkeyPatch, database_error: bool
    ) -> None:
        import litellm

        journal: list = []
        row, instance = self._setup(journal, status="expired", output_file_id=None, error_file_id="file-err-1")
        instance.prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(
            side_effect=RuntimeError("db down") if database_error else None, return_value=0
        )
        logger = self._recording_logger(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))

        with self._provider(journal):
            await instance.check_batch_cost()
            await self._stored(instance)

        assert journal == []
        assert logger.payloads == []
        assert row.batch_processed is False

    @pytest.mark.asyncio
    @pytest.mark.parametrize("has_batch_processed_column, stored", [(True, 1), (False, 0)])
    async def test_a_billed_batch_that_could_not_be_marked_complete_sends_lines_only_if_its_claim_retired_it(
        self, monkeypatch, has_batch_processed_column, stored
    ):
        import litellm

        journal: list = []
        row, instance = self._setup(journal)
        instance._has_batch_processed_column = has_batch_processed_column
        instance.prisma_client.db.litellm_managedobjecttable.update = AsyncMock(side_effect=RuntimeError("db down"))
        logger = self._recording_logger(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))

        with self._provider(journal):
            await instance.check_batch_cost()
            await self._stored(instance)

        assert journal.count("bill") == 1
        assert len(logger.payloads) == stored

    @pytest.mark.asyncio
    async def test_a_batch_whose_output_file_is_gone_still_sends_its_error_lines_after_it_is_retired(self, monkeypatch):
        import litellm

        journal: list = []
        row, instance = self._setup(journal, output_file_id="file-out-deleted", error_file_id="file-err-1")
        logger = self._recording_logger(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))

        with self._provider(journal):
            await instance.check_batch_cost()
            await self._stored(instance)

        assert journal == ["read file-out-deleted", "finalize", "read file-err-1", "read file-input-1", "lines"]
        (payload,) = logger.payloads
        assert [(line["custom_id"], line["status"]) for line in payload["lines"]] == [("ok-1", "failure")]
        assert row.batch_processed is True

    @pytest.mark.asyncio
    async def test_a_terminal_batch_with_no_lines_sends_nothing(self, monkeypatch):
        import litellm

        journal: list = []
        _, instance = self._setup(journal, status="failed", output_file_id=None, error_file_id=None)
        logger = self._recording_logger(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))

        with self._provider(journal):
            await instance.check_batch_cost()
            await self._stored(instance)

        assert journal == ["finalize"]
        assert logger.payloads == []

    @pytest.mark.asyncio
    async def test_a_hung_sink_never_holds_up_billing_and_retiring_the_batch(self, monkeypatch):
        import litellm
        from litellm.integrations.custom_logger import CustomLogger

        journal: list = []
        row, instance = self._setup(journal)
        released = asyncio.Event()

        class _HungSink(CustomLogger):
            async def async_log_batch_line_items(self, payload):
                journal.append("lines started")
                await released.wait()
                journal.append("lines")

        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (_HungSink(),))

        async def _sink_started():
            while "lines started" not in journal:
                await asyncio.sleep(0)

        with self._provider(journal):
            await asyncio.wait_for(instance.check_batch_cost(), timeout=5)
            await asyncio.wait_for(_sink_started(), timeout=5)
            billed_while_hung = (row.batch_processed, row.status, list(journal))
            released.set()
            await self._stored(instance)

        assert billed_while_hung == (
            True,
            "complete",
            [
                f"read {_CLAIM_OUTPUT_FILE_ID}",
                "claim",
                "bill",
                f"read {_CLAIM_OUTPUT_FILE_ID}",
                "read file-input-1",
                "lines started",
            ],
        )
        assert journal[-1] == "lines"

    @pytest.mark.asyncio
    async def test_lines_are_dropped_with_an_error_while_a_poll_cycle_of_batches_is_still_waiting(
        self, monkeypatch, caplog
    ):
        import litellm
        from litellm.constants import MAX_OBJECTS_PER_POLL_CYCLE

        journal: list = []
        row, instance = self._setup(journal)
        logger = self._recording_logger(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
        waiting = [asyncio.create_task(asyncio.sleep(3600)) for _ in range(MAX_OBJECTS_PER_POLL_CYCLE)]
        instance._line_item_stores.update(waiting)

        try:
            with self._provider(journal), caplog.at_level(logging.ERROR):
                await instance.check_batch_cost()
        finally:
            for task in waiting:
                task.cancel()

        assert logger.payloads == []
        assert row.batch_processed is True
        assert f"so the line items of batch batch-456 (job {row.id}) were not stored" in caplog.text

    @pytest.mark.asyncio
    async def test_batches_store_their_lines_one_at_a_time_in_the_order_they_were_retired(self, monkeypatch):
        import litellm

        journal: list = []
        _, instance = self._setup(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (self._recording_logger(journal),))
        running: list = []

        async def _store(job, response, model_id, batch_id, *, billed):
            running.append(batch_id)
            journal.append(f"start {batch_id} with {len(running)} running")
            await asyncio.sleep(0.01)
            running.remove(batch_id)

        monkeypatch.setattr(instance, "_store_batch_line_items", _store)
        for batch_id in ("batch-a", "batch-b", "batch-c"):
            instance._store_batch_line_items_in_background(MagicMock(), MagicMock(), "model-123", batch_id, billed=True)
        await self._stored(instance)

        assert journal == [f"start batch-{name} with 1 running" for name in "abc"]

    @pytest.mark.asyncio
    async def test_lines_are_stored_only_once_the_poll_has_billed_and_retired_every_batch_it_found(self, monkeypatch):
        import litellm

        journal: list = []
        _, instance = self._setup(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (self._recording_logger(journal),))

        async def _an_earlier_batch_waits_to_store_its_lines():
            instance._store_batch_line_items_in_background(
                MagicMock(), MagicMock(), "model-123", "batch-earlier", billed=True
            )
            await asyncio.sleep(0.01)

        async def _store(job, response, model_id, batch_id, *, billed):
            journal.append(f"store {batch_id}")

        monkeypatch.setattr(instance, "_cleanup_stale_managed_objects", _an_earlier_batch_waits_to_store_its_lines)
        monkeypatch.setattr(instance, "_store_batch_line_items", _store)
        with self._provider(journal):
            await instance.check_batch_cost()
            await self._stored(instance)

        assert journal == [
            f"read {_CLAIM_OUTPUT_FILE_ID}",
            "claim",
            "bill",
            "store batch-earlier",
            "store batch-456",
        ]

    @pytest.mark.asyncio
    async def test_an_output_file_that_cannot_be_read_again_leaves_the_batch_billed_and_stores_nothing(
        self, monkeypatch, caplog
    ):
        import litellm
        from litellm.batches import batch_utils
        from litellm.exceptions import NotFoundError

        journal: list = []
        row, instance = self._setup(journal)
        logger = self._recording_logger(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
        fetch = batch_utils.fetch_batch_managed_file_content

        async def _output_gone_after_billing(file_id, **kwargs):
            if file_id == _CLAIM_OUTPUT_FILE_ID:
                raise NotFoundError(message="No file found.", model="gpt-4", llm_provider="openai")
            return await fetch(file_id, **kwargs)

        monkeypatch.setattr(batch_utils, "fetch_batch_managed_file_content", _output_gone_after_billing)
        with self._provider(journal), caplog.at_level(logging.ERROR):
            await instance.check_batch_cost()
            await self._stored(instance)

        assert journal == [f"read {_CLAIM_OUTPUT_FILE_ID}", "claim", "bill"]
        assert logger.payloads == []
        assert (row.batch_processed, row.status) == (True, "complete")
        assert f"the line items of batch batch-456 (job {row.id}) were not stored" in caplog.text

    @pytest.mark.asyncio
    async def test_a_deployment_removed_before_the_lines_are_stored_leaves_the_batch_billed_and_logs_an_error(
        self, monkeypatch, caplog
    ):
        import litellm

        journal: list = []
        row, instance = self._setup(journal)
        logger = self._recording_logger(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
        deployment = instance.llm_router.get_deployment.return_value
        instance.llm_router.get_deployment = MagicMock(
            side_effect=lambda **_: None if instance._poll_idle.is_set() else deployment
        )

        with self._provider(journal), caplog.at_level(logging.ERROR):
            await instance.check_batch_cost()
            await self._stored(instance)

        assert journal == [f"read {_CLAIM_OUTPUT_FILE_ID}", "claim", "bill"]
        assert logger.payloads == []
        assert (row.batch_processed, row.status) == (True, "complete")
        assert f"the line items of batch batch-456 (job {row.id}) were not stored" in caplog.text

    @pytest.mark.asyncio
    async def test_a_batch_bigger_than_one_part_reaches_the_callbacks_as_numbered_parts(self, monkeypatch):
        import litellm
        from litellm_enterprise.proxy.common_utils import check_batch_cost

        journal: list = []
        _, instance = self._setup(journal)
        logger = self._recording_logger(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
        monkeypatch.setattr(litellm, "turn_off_message_logging", False)
        monkeypatch.setattr(check_batch_cost, "BATCH_LINE_ITEMS_MAX_PART_BYTES", 1)
        monkeypatch.setattr(type(self), "_INPUT", self._INPUT + self._INPUT.replace(b"ok-1", b"ok-2"))
        monkeypatch.setattr(type(self), "_OUTPUT", self._OUTPUT + self._OUTPUT.replace(b"ok-1", b"ok-2"))

        with self._provider(journal):
            await instance.check_batch_cost()
            await self._stored(instance)

        assert [(payload["part"], [line["custom_id"] for line in payload["lines"]]) for payload in logger.payloads] == [
            (0, ["ok-1"]),
            (1, ["ok-2"]),
        ]
        assert [payload["lines"][0]["request"]["custom_id"] for payload in logger.payloads] == ["ok-1", "ok-2"]

    @pytest.mark.asyncio
    async def test_an_oauth2_creators_token_reaches_the_callbacks_only_as_its_hash(self, monkeypatch):
        import litellm

        journal: list = []
        row, instance = self._setup(journal)
        row.api_key = "ya29.opaque-oauth2-access-token"
        logger = self._recording_logger(journal)
        monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,))
        monkeypatch.setattr(litellm, "turn_off_message_logging", False)

        with self._provider(journal):
            await instance.check_batch_cost()
            await self._stored(instance)

        (payload,) = logger.payloads
        (line,) = payload["lines"]
        assert line["metadata"]["user_api_key_hash"] == sha256(b"ya29.opaque-oauth2-access-token").hexdigest()
        assert line["request"] == json.loads(self._INPUT)
        assert "ya29.opaque-oauth2-access-token" not in json.dumps(payload)


class TestCreatorTurnedOffMessageLogging:
    """Batch line items drop the request and response when the creating key or team turned off message logging,
    when the proxy did, or when the creator can no longer be read."""

    @staticmethod
    def _instance(key_row=None, team_row=None, error: Exception | None = None):
        from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost

        prisma = MagicMock()
        prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=key_row, side_effect=error)
        prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=team_row)
        return CheckBatchCost(proxy_logging_obj=MagicMock(), prisma_client=prisma, llm_router=MagicMock())

    @staticmethod
    def _job(api_key=sha256(b"sk-alice").hexdigest(), team_id=None, created_by="alice"):
        from types import SimpleNamespace

        return SimpleNamespace(id="job-1", api_key=api_key, team_id=team_id, created_by=created_by)

    @staticmethod
    def _logging(value: str) -> dict:
        return {
            "logging": [
                {
                    "callback_name": "s3_v2",
                    "callback_type": "success",
                    "callback_vars": {"turn_off_message_logging": value},
                }
            ]
        }

    @pytest.mark.asyncio
    @pytest.mark.parametrize(
        ("key_value", "proxy_wide", "expected"),
        [("true", False, True), ("false", True, False), (None, True, True), (None, False, False)],
    )
    async def test_the_key_setting_wins_and_the_proxy_setting_fills_in(
        self, monkeypatch, key_value, proxy_wide, expected
    ):
        from types import SimpleNamespace

        import litellm

        monkeypatch.setattr(litellm, "turn_off_message_logging", proxy_wide)
        metadata = self._logging(key_value) if key_value is not None else {}
        instance = self._instance(key_row=SimpleNamespace(metadata=metadata))

        assert await instance._creator_turned_off_message_logging(self._job()) is expected

    @pytest.mark.asyncio
    async def test_a_team_that_turned_off_message_logging_covers_its_keys(self, monkeypatch):
        from types import SimpleNamespace

        import litellm

        monkeypatch.setattr(litellm, "turn_off_message_logging", False)
        instance = self._instance(
            key_row=SimpleNamespace(metadata={}), team_row=SimpleNamespace(metadata=self._logging("true"))
        )

        assert await instance._creator_turned_off_message_logging(self._job(team_id="team-alpha")) is True

    @pytest.mark.asyncio
    @pytest.mark.parametrize(
        ("job_fields", "key_row", "team_row"),
        [
            ({}, None, None),
            ({"team_id": "team-gone"}, "present", None),
            ({"api_key": None, "team_id": "team-gone"}, None, None),
        ],
        ids=["deleted-key", "deleted-team", "deleted-team-of-a-batch-created-without-a-key"],
    )
    async def test_a_creator_that_is_gone_counts_as_turned_off(self, monkeypatch, job_fields, key_row, team_row):
        from types import SimpleNamespace

        import litellm

        monkeypatch.setattr(litellm, "turn_off_message_logging", False)
        instance = self._instance(key_row=SimpleNamespace(metadata={}) if key_row else None, team_row=team_row)

        assert await instance._creator_turned_off_message_logging(self._job(**job_fields)) is True

    @pytest.mark.asyncio
    @pytest.mark.parametrize("api_key", [None, "opaque-oauth2-access-token"], ids=["jwt", "oauth2"])
    @pytest.mark.parametrize(
        ("team_value", "proxy_wide", "expected"),
        [(None, False, False), (None, True, True), ("true", False, True), ("false", True, False)],
    )
    async def test_a_batch_created_without_a_virtual_key_follows_its_team_and_the_proxy_setting(
        self, monkeypatch, api_key, team_value, proxy_wide, expected
    ):
        from types import SimpleNamespace

        import litellm

        monkeypatch.setattr(litellm, "turn_off_message_logging", proxy_wide)
        team_row = SimpleNamespace(metadata=self._logging(team_value)) if team_value is not None else None
        instance = self._instance(team_row=team_row)
        job = self._job(api_key=api_key, team_id="team-alpha" if team_row is not None else None, created_by="alice")

        assert await instance._creator_turned_off_message_logging(job) is expected
        instance.prisma_client.db.litellm_verificationtoken.find_unique.assert_not_called()

    @pytest.mark.asyncio
    async def test_a_failed_lookup_counts_as_turned_off(self, monkeypatch):
        import litellm

        monkeypatch.setattr(litellm, "turn_off_message_logging", False)
        instance = self._instance(error=Exception("db down"))

        assert await instance._creator_turned_off_message_logging(self._job()) is True

    @pytest.mark.asyncio
    @pytest.mark.parametrize("api_key", ["litellm_proxy_master_key", "cli-session-alice"])
    async def test_the_master_key_and_cli_sessions_have_no_key_row_to_read(self, monkeypatch, api_key):
        import litellm

        monkeypatch.setattr(litellm, "turn_off_message_logging", False)
        instance = self._instance()

        assert await instance._creator_turned_off_message_logging(self._job(api_key=api_key)) is False
        instance.prisma_client.db.litellm_verificationtoken.find_unique.assert_not_called()


@pytest.mark.asyncio
async def test_old_completed_batch_remains_available_for_costing() -> None:
    row: Final = _FakeManagedObjectRow()
    row.status = "completed"
    row.created_at = datetime(2000, 1, 1, tzinfo=timezone.utc)
    prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
    poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router())

    await poller._cleanup_stale_managed_objects()

    assert row.batch_processed is False
    assert row.status == "completed"


@pytest.mark.asyncio
@pytest.mark.parametrize("progress", ["claimed", "refreshed"])
async def test_stale_retirement_preserves_concurrent_progress(progress: str) -> None:
    row: Final = _FakeManagedObjectRow()
    selected: Final = copy(row)
    if progress == "claimed":
        row.batch_processed = True
    row.updated_at = datetime(2000, 1, 2, tzinfo=timezone.utc)
    prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
    poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router())

    await poller._retire_job(selected, "provider batch disappeared")

    assert row.batch_processed is (progress == "claimed")
    assert row.updated_at > selected.updated_at


@pytest.mark.asyncio
@pytest.mark.parametrize("progress", ["claimed", "refreshed"])
async def test_stale_terminal_response_preserves_concurrent_progress(progress: str) -> None:
    from litellm.types.utils import LiteLLMBatch

    row: Final = _FakeManagedObjectRow()
    selected: Final = copy(row)
    if progress == "claimed":
        row.batch_processed = True
    row.updated_at = datetime(2000, 1, 2, tzinfo=timezone.utc)
    row.file_object = '{"revision":"fresh"}'
    prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
    poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router())
    response: Final = LiteLLMBatch(
        id="batch-456",
        completion_window="24h",
        created_at=1,
        endpoint="/v1/chat/completions",
        input_file_id="file-input-1",
        object="batch",
        status="failed",
    )

    assert await poller._finalize_unbilled_terminal_job(selected, response) is False

    assert row.file_object == '{"revision":"fresh"}'
    assert row.status == "in_progress"
    assert row.batch_processed is (progress == "claimed")


@pytest.mark.asyncio
@pytest.mark.parametrize("legacy", [False, True])
async def test_permanently_lost_batch_retires_unchanged_row(legacy: bool) -> None:
    row: Final = _FakeManagedObjectRow()
    prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
    poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router())
    poller._has_batch_processed_column = not legacy

    await poller._retire_job(row, "provider batch disappeared")

    assert row.batch_processed is (not legacy)
    assert row.status == ("stale_expired" if legacy else "in_progress")


@pytest.mark.asyncio
async def test_failed_retirement_keeps_batch_available() -> None:
    row: Final = _FakeManagedObjectRow()
    prisma: Final = TestMultiPodBatchCostClaim._prisma(row, [])
    prisma.db.litellm_managedobjecttable.update_many = AsyncMock(side_effect=RuntimeError("database unavailable"))
    poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router())

    await poller._retire_job(row, "provider batch disappeared")

    assert row.batch_processed is False
    assert row.status == "in_progress"


@pytest.mark.parametrize("resource_id", ("batch-123", "file-123", "gs://bucket/job/predictions.jsonl"))
@pytest.mark.parametrize("other_id", ("{id}4", "prefix-{id}", "{id}/child", "{id}.backup", "{id}?generation=2"))
def test_another_resource_in_a_404_is_not_permanent_loss(resource_id: str, other_id: str) -> None:
    from litellm.exceptions import NotFoundError
    from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost

    error: Final = NotFoundError(
        message=f"No object found with id '{other_id.format(id=resource_id)}'.", model="test", llm_provider="openai"
    )
    assert CheckBatchCost._is_batch_gone_at_provider(error, resource_id) is False
    assert CheckBatchCost._is_output_file_gone_at_provider(error, resource_id) is False


@pytest.mark.parametrize("resource_id", ("batch-123", "file-123", "gs://bucket/job/predictions.jsonl"))
@pytest.mark.parametrize("message", ("No object found with id '{id}'.", 'File "{id}" does not exist', "{id} not found"))
def test_a_404_naming_the_exact_resource_is_permanent_loss(resource_id: str, message: str) -> None:
    from litellm.exceptions import NotFoundError
    from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost

    error: Final = NotFoundError(message=message.format(id=resource_id), model="test", llm_provider="openai")
    assert CheckBatchCost._is_batch_gone_at_provider(error, resource_id) is True
    assert CheckBatchCost._is_output_file_gone_at_provider(error, resource_id) is True


@pytest.mark.parametrize("output_file_id", (None, ""))
def test_a_404_without_an_output_identity_remains_retryable(output_file_id: str | None) -> None:
    from litellm.exceptions import NotFoundError
    from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost

    error: Final = NotFoundError(message="Output file not found", model="test", llm_provider="openai")
    assert CheckBatchCost._is_output_file_gone_at_provider(error, output_file_id) is False


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "kind", ["sync", "async", "custom", "overlapping", "general", "mixed", "sync_only", "executor_shutdown"]
)
@pytest.mark.parametrize("initialized", [False, True])
@pytest.mark.parametrize("claimed", [False, True])
async def test_batch_cost_reaches_configured_success_callbacks(
    kind: Literal["sync", "async", "custom", "overlapping", "general", "mixed", "sync_only", "executor_shutdown"],
    initialized: bool,
    claimed: bool,
) -> None:
    import litellm
    from litellm.integrations.custom_logger import CustomLogger
    from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
    from litellm.types.utils import LiteLLMBatch
    from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost

    events: Final[SimpleQueue[tuple[str, object, int, int]]] = SimpleQueue()

    def record(
        kwargs: dict[str, object],
        response_obj: object,
        start_time: datetime,
        end_time: datetime,
        recipient: str = "function",
    ) -> None:
        if kwargs.get("call_type") != "aretrieve_batch":
            return
        assert isinstance(response_obj, LiteLLMBatch)
        assert response_obj.usage is not None
        events.put(
            (recipient, kwargs["response_cost"], response_obj.usage.prompt_tokens, response_obj.usage.completion_tokens)
        )

    async def async_record(
        kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
    ) -> None:
        record(kwargs, response_obj, start_time, end_time)
        if kind == "executor_shutdown" and kwargs.get("call_type") == "aretrieve_batch":
            executor.shutdown(wait=True)

    class BatchRecorder(CustomLogger):
        def log_success_event(
            self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
        ) -> None:
            record(kwargs, response_obj, start_time, end_time, "custom")

        async def async_log_success_event(
            self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
        ) -> None:
            record(kwargs, response_obj, start_time, end_time, "custom")

    class SyncRecorder(CustomLogger):
        def log_success_event(
            self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
        ) -> None:
            record(kwargs, response_obj, start_time, end_time, "custom")

    callback: Final = SyncRecorder() if kind == "sync_only" else BatchRecorder()
    match kind:
        case "sync":
            litellm.logging_callback_manager.add_litellm_success_callback(record)
        case "async":
            litellm.logging_callback_manager.add_litellm_success_callback(async_record)
        case "executor_shutdown":
            litellm.logging_callback_manager.add_litellm_success_callback(async_record)
            litellm.logging_callback_manager.add_litellm_success_callback(record)
        case "custom" | "overlapping" | "sync_only":
            litellm.logging_callback_manager.add_litellm_success_callback(callback)
            if kind == "overlapping":
                litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
        case "general":
            litellm.logging_callback_manager.add_litellm_callback(callback)
        case "mixed":
            litellm.logging_callback_manager.add_litellm_success_callback(callback)
            litellm.logging_callback_manager.add_litellm_success_callback(record)

    model: Final = "gpt-6.1-sol"
    credentials: Final = {
        "api_key": "synthetic-key",
        "api_base": "https://batch.invalid/v1",
        "custom_llm_provider": "openai",
    }
    deployment: Final = SimpleNamespace(
        litellm_params=LiteLLM_Params(model=f"openai/{model}", custom_llm_provider="openai", api_base=None),
        model_name="batch-group",
    )
    prisma: Final = MagicMock()
    prisma.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=int(claimed))
    checker: Final = CheckBatchCost(
        proxy_logging_obj=SimpleNamespace(get_proxy_hook=lambda _: None),
        prisma_client=prisma,
        llm_router=SimpleNamespace(
            get_deployment_credentials_with_provider=lambda _: credentials,
            get_deployment=lambda **_: deployment,
        ),
    )
    job: Final = SimpleNamespace(
        id="job",
        unified_object_id="batch-record",
        status="in_progress",
        file_object={},
        created_by=None,
        team_id=None,
        api_key=None,
        org_id=None,
        request_tags=None,
    )
    response: Final = LiteLLMBatch(
        id="batch-record",
        completion_window="24h",
        created_at=1,
        endpoint="/v1/chat/completions",
        input_file_id="input",
        object="batch",
        status="completed",
        output_file_id="output",
    )
    output: Final = json.dumps(
        {
            "id": "line",
            "response": {
                "status_code": 200,
                "body": {
                    "model": model,
                    "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
                    "choices": [
                        {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}
                    ],
                },
            },
        }
    ).encode()
    with (
        ThreadPoolExecutor(max_workers=1) as executor,
        patch("litellm.litellm_core_utils.litellm_logging.executor", executor),
        respx.mock(assert_all_called=True) as transport,
    ):
        transport.get("https://batch.invalid/v1/files/output/content").respond(200, content=output)
        if initialized:
            await litellm.afile_content(file_id="output", **credentials)
            await GLOBAL_LOGGING_WORKER.flush()
        result: Final = await checker._track_completed_batch_cost(job, response, "deployment", "batch-record", None)
        assert result == ((model, "openai") if claimed else None)
        if kind == "executor_shutdown" and claimed:
            prisma.db.litellm_managedobjecttable.update_many.assert_awaited_once()
        await GLOBAL_LOGGING_WORKER.flush()
    recipients: Final = (
        ("custom", "function")
        if kind == "mixed"
        else ("custom",)
        if kind in ("custom", "overlapping", "general", "sync_only")
        else ("function",)
    )
    expected: Final = recipients if claimed else ()
    assert events.qsize() == len(expected)
    received: Final = tuple(events.get_nowait() for _ in expected)
    assert tuple(sorted(event[0] for event in received)) == expected
    for _, cost, prompt_tokens, completion_tokens in received:
        assert isinstance(cost, float) and cost > 0
        assert (prompt_tokens, completion_tokens) == (10, 5)