Skip to content

A programming framework for agentic AI

15.7 KB
Raw Blame
from pathlib import Path

import pytest
from autogen_core.memory import MemoryContent, MemoryMimeType
from autogen_core.model_context import BufferedChatCompletionContext
from autogen_core.models import UserMessage
from autogen_ext.memory.chromadb import (
    ChromaDBVectorMemory,
    CustomEmbeddingFunctionConfig,
    DefaultEmbeddingFunctionConfig,
    HttpChromaDBVectorMemoryConfig,
    OpenAIEmbeddingFunctionConfig,
    PersistentChromaDBVectorMemoryConfig,
    SentenceTransformerEmbeddingFunctionConfig,
)

# Skip all tests if ChromaDB is not available
try:
    import chromadb  # pyright: ignore[reportUnusedImport]
except ImportError:
    pytest.skip("ChromaDB not available", allow_module_level=True)


@pytest.fixture
def base_config(tmp_path: Path) -> PersistentChromaDBVectorMemoryConfig:
    """Create base configuration without score threshold."""
    return PersistentChromaDBVectorMemoryConfig(
        collection_name="test_collection", allow_reset=True, k=3, persistence_path=str(tmp_path / "chroma_db")
    )


@pytest.fixture
def strict_config(tmp_path: Path) -> PersistentChromaDBVectorMemoryConfig:
    """Create configuration with strict score threshold."""
    return PersistentChromaDBVectorMemoryConfig(
        collection_name="test_collection",
        allow_reset=True,
        k=3,
        score_threshold=0.8,  # High threshold for strict matching
        persistence_path=str(tmp_path / "chroma_db_strict"),
    )


@pytest.fixture
def lenient_config(tmp_path: Path) -> PersistentChromaDBVectorMemoryConfig:
    """Create configuration with lenient score threshold."""
    return PersistentChromaDBVectorMemoryConfig(
        collection_name="test_collection",
        allow_reset=True,
        k=3,
        score_threshold=0.0,  # No threshold for maximum retrieval
        persistence_path=str(tmp_path / "chroma_db_lenient"),
    )


@pytest.mark.asyncio
async def test_basic_workflow(base_config: PersistentChromaDBVectorMemoryConfig) -> None:
    """Test basic memory operations with default threshold."""
    memory = ChromaDBVectorMemory(config=base_config)
    await memory.clear()

    await memory.add(
        MemoryContent(
            content="Paris is known for the Eiffel Tower and amazing cuisine.",
            mime_type=MemoryMimeType.TEXT,
            metadata={"category": "city", "country": "France"},
        )
    )

    results = await memory.query("Tell me about Paris")
    assert len(results.results) > 0
    assert any("Paris" in str(r.content) for r in results.results)
    assert all(isinstance(r.metadata.get("score"), float) for r in results.results if r.metadata)

    await memory.close()


@pytest.mark.asyncio
async def test_content_types(lenient_config: PersistentChromaDBVectorMemoryConfig) -> None:
    """Test different content types with lenient matching."""
    memory = ChromaDBVectorMemory(config=lenient_config)
    await memory.clear()

    # Test text content
    text_content = MemoryContent(content="Simple text content for testing", mime_type=MemoryMimeType.TEXT)
    await memory.add(text_content)

    # Test JSON content
    json_data = {"key": "value", "number": 42}
    json_content = MemoryContent(content=json_data, mime_type=MemoryMimeType.JSON)
    await memory.add(json_content)

    # Query for text content
    results = await memory.query("simple text content")
    assert len(results.results) > 0
    assert any("Simple text content" in str(r.content) for r in results.results)

    # Query for JSON content
    results = await memory.query("value")
    result_contents = [str(r.content).lower() for r in results.results]
    assert any("value" in content for content in result_contents)

    await memory.close()


@pytest.mark.asyncio
async def test_strict_matching(strict_config: PersistentChromaDBVectorMemoryConfig) -> None:
    """Test matching behavior with high score threshold."""
    memory = ChromaDBVectorMemory(config=strict_config)
    await memory.clear()

    await memory.add(
        MemoryContent(content="Specific technical details about quantum computing", mime_type=MemoryMimeType.TEXT)
    )

    # Exact query should match
    exact_results = await memory.query("quantum computing details")
    assert len(exact_results.results) > 0
    assert all(
        result.metadata and result.metadata.get("score", 0) >= strict_config.score_threshold
        for result in exact_results.results
    )

    # Unrelated query should not match due to high threshold
    unrelated_results = await memory.query("recipe for cake")
    assert len(unrelated_results.results) == 0

    await memory.close()


@pytest.mark.asyncio
async def test_metadata_handling(base_config: PersistentChromaDBVectorMemoryConfig) -> None:
    """Test metadata handling with default threshold."""
    memory = ChromaDBVectorMemory(config=base_config)
    await memory.clear()

    test_content = "Test content with specific metadata"
    content = MemoryContent(
        content=test_content,
        mime_type=MemoryMimeType.TEXT,
        metadata={"test_category": "test", "test_priority": 1, "test_weight": 0.5, "test_verified": True},
    )
    await memory.add(content)

    results = await memory.query(test_content)
    assert len(results.results) > 0
    result = results.results[0]

    assert result.metadata is not None
    assert result.metadata.get("test_category") == "test"
    assert result.metadata.get("test_priority") == 1
    assert isinstance(result.metadata.get("test_weight"), float)
    assert result.metadata.get("test_verified") is True

    await memory.close()


@pytest.mark.asyncio
async def test_error_handling(base_config: PersistentChromaDBVectorMemoryConfig) -> None:
    """Test error cases with default threshold."""
    memory = ChromaDBVectorMemory(config=base_config)
    await memory.clear()

    with pytest.raises(ValueError, match="Unsupported content type"):
        await memory.add(MemoryContent(content=b"binary data", mime_type=MemoryMimeType.BINARY))

    with pytest.raises(ValueError, match="JSON content must be a dict"):
        await memory.add(MemoryContent(content="not a dict", mime_type=MemoryMimeType.JSON))

    await memory.close()


@pytest.mark.asyncio
async def test_initialization(base_config: PersistentChromaDBVectorMemoryConfig) -> None:
    """Test initialization with default threshold."""
    memory = ChromaDBVectorMemory(config=base_config)

    # Test that the collection_name property returns the expected value
    # This implicitly tests that initialization succeeds
    assert memory.collection_name == "test_collection"

    # Add something to verify the collection is working
    test_content = MemoryContent(content="Test initialization content", mime_type=MemoryMimeType.TEXT)
    await memory.add(test_content)

    # Verify we can query the added content
    results = await memory.query("Test initialization")
    assert len(results.results) > 0

    # Use the public reset method
    await memory.reset()

    # Verify the reset worked by checking that the previous content is gone
    results_after_reset = await memory.query("Test initialization")
    assert len(results_after_reset.results) == 0

    # Add new content to verify re-initialization happened automatically
    new_content = MemoryContent(content="New test content after reset", mime_type=MemoryMimeType.TEXT)
    await memory.add(new_content)

    # Verify we can query the new content
    new_results = await memory.query("New test")
    assert len(new_results.results) > 0

    await memory.close()


@pytest.mark.asyncio
async def test_model_context_update(base_config: PersistentChromaDBVectorMemoryConfig) -> None:
    """Test updating model context with retrieved memories."""
    memory = ChromaDBVectorMemory(config=base_config)
    await memory.clear()

    # Add content to memory
    await memory.add(
        MemoryContent(
            content="Jupiter is the largest planet in our solar system.",
            mime_type=MemoryMimeType.TEXT,
            metadata={"category": "astronomy"},
        )
    )

    # Create a model context with a message
    context = BufferedChatCompletionContext(buffer_size=5)
    await context.add_message(UserMessage(content="Tell me about Jupiter", source="user"))

    # Update context with memory
    result = await memory.update_context(context)

    # Verify results
    assert len(result.memories.results) > 0
    assert any("Jupiter" in str(r.content) for r in result.memories.results)

    # Verify context was updated
    messages = await context.get_messages()
    assert len(messages) > 1  # Should have the original message plus the memory content

    await memory.close()


@pytest.mark.asyncio
async def test_component_serialization(base_config: PersistentChromaDBVectorMemoryConfig) -> None:
    """Test serialization and deserialization of the component."""
    memory = ChromaDBVectorMemory(config=base_config)

    # Serialize
    memory_config = memory.dump_component()
    assert memory_config.config["collection_name"] == base_config.collection_name

    # Deserialize
    loaded_memory = ChromaDBVectorMemory.load_component(memory_config)

    assert isinstance(loaded_memory, ChromaDBVectorMemory)

    await memory.close()
    await loaded_memory.close()


@pytest.mark.asyncio
def test_http_config(tmp_path: Path) -> None:
    """Test HTTP ChromaDB configuration."""
    config = HttpChromaDBVectorMemoryConfig(
        collection_name="test_http",
        host="localhost",
        port=8000,
        ssl=False,
        headers={"Authorization": "Bearer test-token"},
    )

    assert config.client_type == "http"
    assert config.host == "localhost"
    assert config.port == 8000
    assert config.ssl is False
    assert config.headers == {"Authorization": "Bearer test-token"}


# ============================================================================
# Embedding Function Configuration Tests
# ============================================================================


@pytest.mark.asyncio
async def test_default_embedding_function(tmp_path: Path) -> None:
    """Test ChromaDB memory with default embedding function."""
    config = PersistentChromaDBVectorMemoryConfig(
        collection_name="test_default_embedding",
        allow_reset=True,
        persistence_path=str(tmp_path / "chroma_db_default"),
        embedding_function_config=DefaultEmbeddingFunctionConfig(),
    )

    memory = ChromaDBVectorMemory(config=config)
    await memory.clear()

    # Add test content
    await memory.add(
        MemoryContent(
            content="Default embedding function test content",
            mime_type=MemoryMimeType.TEXT,
            metadata={"test": "default_embedding"},
        )
    )

    # Query and verify
    results = await memory.query("default embedding test")
    assert len(results.results) > 0
    assert any("Default embedding" in str(r.content) for r in results.results)

    await memory.close()


@pytest.mark.asyncio
async def test_sentence_transformer_embedding_function(tmp_path: Path) -> None:
    """Test ChromaDB memory with SentenceTransformer embedding function."""
    config = PersistentChromaDBVectorMemoryConfig(
        collection_name="test_st_embedding",
        allow_reset=True,
        persistence_path=str(tmp_path / "chroma_db_st"),
        embedding_function_config=SentenceTransformerEmbeddingFunctionConfig(
            model_name="all-MiniLM-L6-v2"  # Use default model for testing
        ),
    )

    memory = ChromaDBVectorMemory(config=config)
    await memory.clear()

    # Add test content
    await memory.add(
        MemoryContent(
            content="SentenceTransformer embedding function test content",
            mime_type=MemoryMimeType.TEXT,
            metadata={"test": "sentence_transformer"},
        )
    )

    # Query and verify
    results = await memory.query("SentenceTransformer embedding test")
    assert len(results.results) > 0
    assert any("SentenceTransformer" in str(r.content) for r in results.results)

    await memory.close()


@pytest.mark.asyncio
async def test_custom_embedding_function(tmp_path: Path) -> None:
    """Test ChromaDB memory with custom embedding function."""
    from collections.abc import Sequence

    class MockEmbeddingFunction:
        def __call__(self, input: Sequence[str]) -> list[list[float]]:
            # Return a batch of embeddings (list of lists)
            return [[0.0] * 384 for _ in input]

    config = PersistentChromaDBVectorMemoryConfig(
        collection_name="test_custom_embedding",
        allow_reset=True,
        persistence_path=str(tmp_path / "chroma_db_custom"),
        embedding_function_config=CustomEmbeddingFunctionConfig(function=MockEmbeddingFunction, params={}),
    )
    memory = ChromaDBVectorMemory(config=config)
    await memory.clear()
    await memory.add(
        MemoryContent(
            content="Custom embedding function test content",
            mime_type=MemoryMimeType.TEXT,
            metadata={"test": "custom_embedding"},
        )
    )
    results = await memory.query("custom embedding test")
    assert len(results.results) > 0
    assert any("Custom embedding" in str(r.content) for r in results.results)
    await memory.close()


@pytest.mark.asyncio
async def test_openai_embedding_function(tmp_path: Path) -> None:
    """Test OpenAI embedding function configuration (without actual API call)."""
    config = PersistentChromaDBVectorMemoryConfig(
        collection_name="test_openai_embedding",
        allow_reset=True,
        persistence_path=str(tmp_path / "chroma_db_openai"),
        embedding_function_config=OpenAIEmbeddingFunctionConfig(
            api_key="test-key", model_name="text-embedding-3-small"
        ),
    )

    # Just test that the config is valid - don't actually try to use OpenAI API
    assert config.embedding_function_config.function_type == "openai"
    assert config.embedding_function_config.api_key == "test-key"
    assert config.embedding_function_config.model_name == "text-embedding-3-small"


@pytest.mark.asyncio
async def test_embedding_function_error_handling(tmp_path: Path) -> None:
    """Test error handling for embedding function configurations."""

    def failing_embedding_function() -> None:
        """A function that raises an error."""
        raise ValueError("Test embedding function error")

    config = PersistentChromaDBVectorMemoryConfig(
        collection_name="test_error_embedding",
        allow_reset=True,
        persistence_path=str(tmp_path / "chroma_db_error"),
        embedding_function_config=CustomEmbeddingFunctionConfig(function=failing_embedding_function, params={}),
    )

    memory = ChromaDBVectorMemory(config=config)

    # Should raise an error when trying to initialize
    with pytest.raises((ValueError, Exception)):  # Catch ValueError or any other exception
        await memory.add(MemoryContent(content="This should fail", mime_type=MemoryMimeType.TEXT))

    await memory.close()


def test_embedding_function_config_validation() -> None:
    """Test validation of embedding function configurations."""

    # Test default config
    default_config = DefaultEmbeddingFunctionConfig()
    assert default_config.function_type == "default"

    # Test SentenceTransformer config
    st_config = SentenceTransformerEmbeddingFunctionConfig(model_name="test-model")
    assert st_config.function_type == "sentence_transformer"
    assert st_config.model_name == "test-model"

    # Test OpenAI config
    openai_config = OpenAIEmbeddingFunctionConfig(api_key="test-key", model_name="test-model")
    assert openai_config.function_type == "openai"
    assert openai_config.api_key == "test-key"
    assert openai_config.model_name == "test-model"

    # Test custom config
    def dummy_function() -> None:
        return None

    custom_config = CustomEmbeddingFunctionConfig(function=dummy_function, params={"test": "value"})
    assert custom_config.function_type == "custom"
    assert custom_config.function == dummy_function
    assert custom_config.params == {"test": "value"}