Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python
9.8 KB
Raw Blame
import os
import tempfile

import pydantic
import pytest

import dspy
from dspy.adapters.types.file import encode_file_to_dict
from dspy.utils.dummies import DummyLM


@pytest.fixture
def sample_text_file():
    with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as tmp_file:
        tmp_file.write("This is a test file.")
        tmp_file_path = tmp_file.name
    yield tmp_file_path
    try:
        os.unlink(tmp_file_path)
    except Exception:
        pass


def count_messages_with_file_pattern(messages):
    pattern = {"type": "file", "file": lambda x: isinstance(x, dict)}

    def check_pattern(obj, pattern):
        if isinstance(pattern, dict):
            if not isinstance(obj, dict):
                return False
            return all(k in obj and check_pattern(obj[k], v) for k, v in pattern.items())
        if callable(pattern):
            return pattern(obj)
        return obj == pattern

    def count_patterns(obj, pattern):
        count = 0
        if check_pattern(obj, pattern):
            count += 1
        if isinstance(obj, dict):
            count += sum(count_patterns(v, pattern) for v in obj.values())
        if isinstance(obj, list | tuple):
            count += sum(count_patterns(v, pattern) for v in obj)
        return count

    return count_patterns(messages, pattern)


def setup_predictor(signature, expected_output):
    lm = DummyLM([expected_output])
    dspy.settings.configure(lm=lm)
    return dspy.Predict(signature), lm


def test_file_from_local_path(sample_text_file):
    file_obj = dspy.File.from_path(sample_text_file)
    assert file_obj.file_data is not None
    assert file_obj.file_data.startswith("data:text/plain;base64,")
    assert file_obj.filename == os.path.basename(sample_text_file)


def test_file_from_path_method(sample_text_file):
    file_obj = dspy.File.from_path(sample_text_file)
    assert file_obj.file_data is not None
    assert file_obj.file_data.startswith("data:text/plain;base64,")
    assert file_obj.filename == os.path.basename(sample_text_file)


def test_file_from_path_with_custom_filename(sample_text_file):
    file_obj = dspy.File.from_path(sample_text_file, filename="custom.txt")
    assert file_obj.file_data is not None
    assert file_obj.file_data.startswith("data:text/plain;base64,")
    assert file_obj.filename == "custom.txt"


def test_file_from_bytes():
    file_bytes = b"Test file content"
    file_obj = dspy.File.from_bytes(file_bytes)
    assert file_obj.file_data is not None
    assert file_obj.file_data.startswith("data:application/octet-stream;base64,")
    assert file_obj.filename is None


def test_file_from_bytes_with_filename():
    file_bytes = b"Test file content"
    file_obj = dspy.File.from_bytes(file_bytes, filename="test.txt")
    assert file_obj.file_data is not None
    assert file_obj.file_data.startswith("data:application/octet-stream;base64,")
    assert file_obj.filename == "test.txt"


def test_file_from_file_id():
    file_obj = dspy.File.from_file_id("file-abc123")
    assert file_obj.file_id == "file-abc123"
    assert file_obj.file_data is None


def test_file_from_file_id_with_filename():
    file_obj = dspy.File.from_file_id("file-abc123", filename="document.pdf")
    assert file_obj.file_id == "file-abc123"
    assert file_obj.filename == "document.pdf"


def test_file_from_dict_with_file_data():
    file_obj = dspy.File(file_data="data:text/plain;base64,dGVzdA==", filename="test.txt")
    assert file_obj.file_data == "data:text/plain;base64,dGVzdA=="
    assert file_obj.filename == "test.txt"


def test_file_from_dict_with_file_id():
    file_obj = dspy.File(file_id="file-xyz789")
    assert file_obj.file_id == "file-xyz789"


def test_file_format_with_file_data():
    file_obj = dspy.File.from_bytes(b"test", filename="test.txt")
    formatted = file_obj.format()
    assert isinstance(formatted, list)
    assert len(formatted) == 1
    assert formatted[0]["type"] == "file"
    assert "file" in formatted[0]
    assert "file_data" in formatted[0]["file"]
    assert "filename" in formatted[0]["file"]


def test_file_format_with_file_id():
    file_obj = dspy.File.from_file_id("file-123")
    formatted = file_obj.format()
    assert formatted[0]["type"] == "file"
    assert formatted[0]["file"]["file_id"] == "file-123"


def test_file_repr_with_file_data():
    file_obj = dspy.File.from_bytes(b"Test content", filename="test.txt")
    repr_str = repr(file_obj)
    assert "DATA_URI" in repr_str
    assert "application/octet-stream" in repr_str
    assert "filename='test.txt'" in repr_str


def test_file_repr_with_file_id():
    file_obj = dspy.File.from_file_id("file-abc", filename="doc.pdf")
    repr_str = repr(file_obj)
    assert "file_id='file-abc'" in repr_str
    assert "filename='doc.pdf'" in repr_str


def test_file_str():
    file_obj = dspy.File.from_bytes(b"test")
    str_repr = str(file_obj)
    assert "<<CUSTOM-TYPE-START-IDENTIFIER>>" in str_repr
    assert "<<CUSTOM-TYPE-END-IDENTIFIER>>" in str_repr


def test_encode_file_to_dict_rejects_path(sample_text_file):
    with pytest.raises(ValueError, match=r"File\.from_path"):
        encode_file_to_dict(sample_text_file)


def test_encode_file_to_dict_from_bytes():
    result = encode_file_to_dict(b"test content")
    assert "file_data" in result
    assert result["file_data"].startswith("data:application/octet-stream;base64,")


def test_file_constructor_from_bytes():
    file_obj = dspy.File(b"test content", filename="test.txt")
    assert file_obj.file_data.startswith("data:application/octet-stream;base64,")
    assert file_obj.filename == "test.txt"


def test_file_constructor_from_data_uri():
    data_uri = "data:text/plain;base64,aGVsbG8="
    file_obj = dspy.File(data_uri)
    assert file_obj.file_data == data_uri


def test_file_constructor_rejects_conflicting_keyword():
    with pytest.raises(TypeError, match="file_data"):
        dspy.File(b"test content", file_data="data:text/plain;base64,aGVsbG8=")


def test_invalid_file_string():
    with pytest.raises(ValueError, match=r"File\.from_path"):
        encode_file_to_dict("https://this_is_not_a_file_path")


def test_invalid_dict():
    with pytest.raises(ValueError, match="must contain at least one"):
        dspy.File(invalid="dict")


def test_file_in_signature(sample_text_file):
    signature = "document: dspy.File -> summary: str"
    expected = {"summary": "This is a summary"}
    predictor, lm = setup_predictor(signature, expected)

    file_obj = dspy.File.from_path(sample_text_file)
    result = predictor(document=file_obj)

    assert result.summary == "This is a summary"
    assert count_messages_with_file_pattern(lm.history[-1]["messages"]) == 1


def test_file_list_in_signature(sample_text_file):
    class FileListSignature(dspy.Signature):
        documents: list[dspy.File] = dspy.InputField()
        summary: str = dspy.OutputField()

    expected = {"summary": "Multiple files"}
    predictor, lm = setup_predictor(FileListSignature, expected)

    files = [
        dspy.File.from_path(sample_text_file),
        dspy.File.from_file_id("file-123"),
    ]
    result = predictor(documents=files)

    assert result.summary == "Multiple files"
    assert count_messages_with_file_pattern(lm.history[-1]["messages"]) == 2


def test_optional_file_field():
    class OptionalFileSignature(dspy.Signature):
        document: dspy.File | None = dspy.InputField()
        output: str = dspy.OutputField()

    predictor, lm = setup_predictor(OptionalFileSignature, {"output": "Hello"})
    result = predictor(document=None)
    assert result.output == "Hello"
    assert count_messages_with_file_pattern(lm.history[-1]["messages"]) == 0


def test_save_load_file_signature(sample_text_file):
    signature = "document: dspy.File -> summary: str"
    file_obj = dspy.File.from_path(sample_text_file)
    examples = [dspy.Example(document=file_obj, summary="Test summary")]

    predictor, lm = setup_predictor(signature, {"summary": "A summary"})
    optimizer = dspy.teleprompt.LabeledFewShot(k=1)
    compiled_predictor = optimizer.compile(student=predictor, trainset=examples, sample=False)

    with tempfile.NamedTemporaryFile(mode="w+", delete=True, suffix=".json") as temp_file:
        compiled_predictor.save(temp_file.name)
        loaded_predictor = dspy.Predict(signature)
        loaded_predictor.load(temp_file.name)

    loaded_predictor(document=dspy.File.from_file_id("file-test"))
    assert count_messages_with_file_pattern(lm.history[-1]["messages"]) == 2


def test_file_frozen():
    file_obj = dspy.File.from_bytes(b"test")
    with pytest.raises((TypeError, ValueError, pydantic.ValidationError)):
        file_obj.file_data = "new_data"


def test_file_with_all_fields():
    file_data_uri = "data:text/plain;base64,dGVzdA=="
    file_obj = dspy.File(file_data=file_data_uri, file_id="file-123", filename="test.txt")
    assert file_obj.file_data == file_data_uri
    assert file_obj.file_id == "file-123"
    assert file_obj.filename == "test.txt"

    formatted = file_obj.format()
    assert formatted[0]["file"]["file_data"] == file_data_uri
    assert formatted[0]["file"]["file_id"] == "file-123"
    assert formatted[0]["file"]["filename"] == "test.txt"


def test_file_path_not_found():
    with pytest.raises(ValueError, match="File not found"):
        dspy.File.from_path("/nonexistent/path/file.txt")


def test_file_custom_mime_type(sample_text_file):
    file_obj = dspy.File.from_path(sample_text_file, mime_type="text/custom")
    assert file_obj.file_data.startswith("data:text/custom;base64,")


def test_file_from_bytes_custom_mime():
    file_obj = dspy.File.from_bytes(b"audio data", mime_type="audio/mp3")
    assert file_obj.file_data.startswith("data:audio/mp3;base64,")


def test_file_data_uri_in_format():
    file_obj = dspy.File.from_bytes(b"test", filename="test.txt", mime_type="text/plain")
    formatted = file_obj.format()
    assert "data:text/plain;base64," in formatted[0]["file"]["file_data"]