Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

7.6 KB
Raw Blame
import json
from typing import Final
from unittest.mock import patch, MagicMock, AsyncMock


import litellm
import pytest
import respx


MOCK_EMBEDDING_RESPONSE = [[0.1, 0.2, 0.3, 0.4, 0.5]]


@pytest.fixture
def mock_embedding_http_handler():
    """Fixture to mock the HTTP handler for embedding tests"""
    with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post:
        mock_response = MagicMock()
        mock_response.raise_for_status.return_value = None
        mock_response.status_code = 200
        mock_response.json.return_value = MOCK_EMBEDDING_RESPONSE
        mock_post.return_value = mock_response
        yield mock_post


@pytest.fixture
def mock_hf_config_fetch():
    """Serve the Hugging Face config.json fetched during cost calculation, so no test leaves the process"""
    with respx.mock(assert_all_called=False) as respx_mock:
        respx_mock.get(url__regex=r"https://huggingface\.co/.*/config\.json").respond(
            json={"max_position_embeddings": 512}
        )
        yield respx_mock


@pytest.fixture
def mock_embedding_async_http_handler():
    """Fixture to mock the async HTTP handler for embedding tests"""
    with patch(
        "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
        new_callable=AsyncMock,
    ) as mock_post:
        mock_response = MagicMock()
        mock_response.raise_for_status.return_value = None
        mock_response.status_code = 200
        mock_response.headers = {"content-type": "application/json"}
        mock_response.json.return_value = MOCK_EMBEDDING_RESPONSE
        mock_post.return_value = mock_response
        yield mock_post


class TestHuggingFaceEmbedding:
    @pytest.fixture(autouse=True)
    def setup(self, mock_embedding_http_handler, mock_embedding_async_http_handler, mock_hf_config_fetch):
        self.mock_get_task_patcher = patch(
            "litellm.llms.huggingface.embedding.handler.get_hf_task_embedding_for_model"
        )
        self.mock_get_task = self.mock_get_task_patcher.start()

        def mock_get_task_side_effect(model, task_type, api_base):
            if task_type is not None:
                return task_type
            return "sentence-similarity"

        self.mock_get_task.side_effect = mock_get_task_side_effect

        self.model = "huggingface/BAAI/bge-m3"
        self.mock_http = mock_embedding_http_handler
        self.mock_async_http = mock_embedding_async_http_handler
        litellm.set_verbose = False

        yield

        self.mock_get_task_patcher.stop()

    def test_input_type_preserved_in_optional_params(self):
        input_text = ["hello world"]

        response = litellm.embedding(
            model=self.model,
            input=input_text,
            input_type="embed",
        )

        self.mock_http.assert_called_once()
        post_call_args = self.mock_http.call_args
        request_data = json.loads(post_call_args[1]["data"])

        # When input_type="embed", it should use simple format {"inputs": [...]}
        # NOT the sentence-similarity format which would require 2+ sentences
        assert "inputs" in request_data
        assert request_data["inputs"] == input_text

        # Should NOT have sentence-similarity format
        assert "source_sentence" not in str(request_data)
        assert "sentences" not in str(request_data)

    def test_embedding_allows_special_token_looking_input(self):
        input_text = ["hello <|fim_prefix|> world"]

        response = litellm.embedding(
            model=self.model,
            input=input_text,
            input_type="embed",
        )

        self.mock_http.assert_called_once()
        post_call_args = self.mock_http.call_args
        request_data = json.loads(post_call_args[1]["data"])

        assert request_data["inputs"] == input_text
        assert response.usage.prompt_tokens > 0
        assert response.usage.total_tokens == response.usage.prompt_tokens

    def test_model_name_with_https_substring_uses_api_base(self):
        api_base = "https://legit.example/embed"

        litellm.embedding(
            model="huggingface/my-https-endpoint",
            input=["hello world"],
            input_type="embed",
            api_base=api_base,
        )

        self.mock_http.assert_called_once()
        called_url = self.mock_http.call_args[0][0]
        assert called_url == api_base

    def test_embedding_with_sentence_similarity_task(self):
        """Test embedding when task type is sentence-similarity (requires 2+ sentences)"""

        similarity_response = {"similarities": [[0, 0.9], [1, 0.8]]}

        self.mock_http.return_value.json.return_value = similarity_response

        # Test with 2+ sentences (required for sentence-similarity)
        input_text = [
            "This is the source sentence",
            "This is sentence one",
            "This is sentence two",
        ]

        response = litellm.embedding(
            model=self.model,
            input=input_text,
            # Use the model's natural task type (sentence-similarity)
        )

        self.mock_http.assert_called_once()
        post_call_args = self.mock_http.call_args
        request_data = json.loads(post_call_args[1]["data"])

        assert "inputs" in request_data
        assert "source_sentence" in request_data["inputs"]
        assert "sentences" in request_data["inputs"]
        assert request_data["inputs"]["source_sentence"] == input_text[0]
        assert request_data["inputs"]["sentences"] == input_text[1:]

    def test_hf_sentence_similarity_transform_keeps_provider_options(self):
        from litellm.llms.huggingface.embedding.handler import HuggingFaceEmbedding

        request_data: Final = HuggingFaceEmbedding()._transform_input(
            input=["source sentence", "candidate sentence"],
            model="BAAI/bge-m3",
            call_type="sync",
            optional_params={"input_type": "sentence-similarity", "min_length": 2},
            embed_url="https://huggingface.example/embeddings",
        )

        assert request_data == {
            "inputs": {"source_sentence": "source sentence", "sentences": ["candidate sentence"]},
            "parameters": {"min_length": 2},
        }


@pytest.mark.parametrize("sync_mode", [True, False])
async def test_hf_embedding_sentence_sim_sends_one_similarity_request(
    sync_mode: bool, monkeypatch: pytest.MonkeyPatch
) -> None:
    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
    monkeypatch.setenv("HUGGINGFACE_API_KEY", "hf-test-token")
    with respx.mock(assert_all_called=False) as hf:
        hf.get(url__regex=r"https://huggingface\.co/api/models/.*").respond(json={"pipeline_tag": "sentence-similarity"})
        hf.get(url__regex=r"https://huggingface\.co/.*/config\.json").respond(json={"max_position_embeddings": 512})
        inference: Final = hf.post(
            "https://router.huggingface.co/hf-inference/pipeline/sentence-similarity/sentence-transformers/TaylorAI/bge-micro-v2"
        ).respond(json=[0.7708950042724609])
        request: Final = {
            "model": "huggingface/sentence-transformers/TaylorAI/bge-micro-v2",
            "input": ["good morning from litellm", "this is another item"],
        }
        response: Final = litellm.embedding(**request) if sync_mode else await litellm.aembedding(**request)

    assert inference.call_count == 1
    assert json.loads(inference.calls.last.request.content) == {
        "inputs": {"source_sentence": "good morning from litellm", "sentences": ["this is another item"]}
    }
    assert inference.calls.last.request.headers["Authorization"] == "Bearer hf-test-token"
    assert response.data == [{"object": "embedding", "index": 0, "embedding": 0.7708950042724609}]