DSPy: The framework for programming—not prompting—language models
Python
Style fixing for tests/ (#8225)
Chen Qiand3a82561f5ec
diff --git a/tests/adapters/test_json_adapter.py b/tests/adapters/test_json_adapter.pyindex 8396126ef..50be7d2eb 100644--- a/tests/adapters/test_json_adapter.py+++ b/tests/adapters/test_json_adapter.py@@ -2,8 +2,7 @@ from unittest import mockimport pydanticimport pytest-from pydantic import create_model-from litellm.utils import ModelResponse, Message, Choices+from litellm.utils import Choices, Message, ModelResponseimport dspy@@ -109,7 +108,7 @@ async def test_json_adapter_async_call():def test_json_adapter_on_pydantic_model():- from litellm.utils import ModelResponse, Message, Choices+ from litellm.utils import Choices, Message, ModelResponseclass User(pydantic.BaseModel):id: int@@ -170,7 +169,7 @@ def test_json_adapter_on_pydantic_model():assert expected_input_structure in content# Assert that system prompt includes output formatting structure- expected_output_structure = ( # noqa: Q000+ expected_output_structure = ("Outputs will be a JSON object with the following fields.\n\n{\n "'"answer": "{answer} # note: the value you produce must adhere to the JSON schema: ''{\\"type\\": \\"object\\", \\"properties\\": {\\"analysis\\": {\\"type\\": \\"string\\", \\"title\\": '@@ -184,7 +183,7 @@ def test_json_adapter_on_pydantic_model():assert user_message_content is not None# Assert that the user input data is formatted correctly- expected_input_data = ( # noqa: Q000+ expected_input_data = ('[[ ## user ## ]]\n{"id": 5, "name": "name_test", "email": "email_test"}\n\n[[ ## question ## ]]\n'"What is the capital of France?\n\n")diff --git a/tests/adapters/test_two_step_adapter.py b/tests/adapters/test_two_step_adapter.pyindex cf83c3dbd..c7a9460a3 100644--- a/tests/adapters/test_two_step_adapter.py+++ b/tests/adapters/test_two_step_adapter.py@@ -1,4 +1,5 @@from unittest import mock+import pytestimport dspy@@ -9,7 +10,7 @@ def test_two_step_adapter_call():question: str = dspy.InputField(desc="The math question to solve")solution: str = dspy.OutputField(desc="Step by step solution")answer: float = dspy.OutputField(desc="The final numerical answer")-+program = dspy.Predict(TestSignature)mock_main_lm = mock.MagicMock(spec=dspy.LM)@@ -30,7 +31,7 @@ def test_two_step_adapter_call():dspy.configure(lm=mock_main_lm, adapter=dspy.TwoStepAdapter(extraction_model=mock_extraction_lm))result = program(question="What is 5 + 7?")-+assert result.answer == 12# main LM call@@ -59,22 +60,23 @@ def test_two_step_adapter_call():# assert first messageassert call_kwargs["messages"][0]["role"] == "system"content = call_kwargs["messages"][0]["content"]- assert "`text` (str)" in content- assert "`solution` (str)" in content- assert "`answer` (float)" in content+ assert "`text` (str)" in content+ assert "`solution` (str)" in content+ assert "`answer` (float)" in content# assert second messageassert call_kwargs["messages"][1]["role"] == "user"content = call_kwargs["messages"][1]["content"]assert "text from main LM" in content+@pytest.mark.asyncioasync def test_two_step_adapter_async_call():class TestSignature(dspy.Signature):question: str = dspy.InputField(desc="The math question to solve")solution: str = dspy.OutputField(desc="Step by step solution")answer: float = dspy.OutputField(desc="The final numerical answer")-+program = dspy.Predict(TestSignature)mock_main_lm = mock.MagicMock(spec=dspy.LM)@@ -95,7 +97,7 @@ async def test_two_step_adapter_async_call():dspy.configure(lm=mock_main_lm, adapter=dspy.TwoStepAdapter(extraction_model=mock_extraction_lm))result = await program.acall(question="What is 5 + 7?")-+assert result.answer == 12# main LM call@@ -124,37 +126,40 @@ async def test_two_step_adapter_async_call():# assert first messageassert call_kwargs["messages"][0]["role"] == "system"content = call_kwargs["messages"][0]["content"]- assert "`text` (str)" in content- assert "`solution` (str)" in content- assert "`answer` (float)" in content+ assert "`text` (str)" in content+ assert "`solution` (str)" in content+ assert "`answer` (float)" in content# assert second messageassert call_kwargs["messages"][1]["role"] == "user"content = call_kwargs["messages"][1]["content"]assert "text from main LM" in content+def test_two_step_adapter_parse():class ComplexSignature(dspy.Signature):input_text: str = dspy.InputField()tags: list[str] = dspy.OutputField(desc="List of relevant tags")confidence: float = dspy.OutputField(desc="Confidence score")-+first_response = "main LM response"-+mock_extraction_lm = mock.MagicMock(spec=dspy.LM)- mock_extraction_lm.return_value = ["""+ mock_extraction_lm.return_value = [+ """{"tags": ["AI", "deep learning", "neural networks"],"confidence": 0.87- }- """]+ }+ """+ ]mock_extraction_lm.kwargs = {"temperature": 1.0}mock_extraction_lm.model = "openai/gpt-4o"adapter = dspy.TwoStepAdapter(mock_extraction_lm)dspy.configure(adapter=adapter, lm=mock_extraction_lm)result = adapter.parse(ComplexSignature, first_response)-+assert result["tags"] == ["AI", "deep learning", "neural networks"]assert result["confidence"] == 0.87@@ -163,7 +168,7 @@ def test_two_step_adapter_parse_errors():class TestSignature(dspy.Signature):question: str = dspy.InputField()answer: str = dspy.OutputField()-+first_response = "main LM response"mock_extraction_lm = mock.MagicMock(spec=dspy.LM)@@ -172,6 +177,6 @@ def test_two_step_adapter_parse_errors():mock_extraction_lm.model = "openai/gpt-4o"adapter = dspy.TwoStepAdapter(mock_extraction_lm)-+with pytest.raises(ValueError, match="Failed to parse response"):- adapter.parse(TestSignature, first_response)+ adapter.parse(TestSignature, first_response)diff --git a/tests/caching/test_litellm_cache.py b/tests/caching/test_litellm_cache.pyindex b2414b5ba..0bf881ed7 100644--- a/tests/caching/test_litellm_cache.py+++ b/tests/caching/test_litellm_cache.py@@ -7,7 +7,7 @@ from unittest.mock import patchimport pytestimport dspy-from tests.test_utils.server import litellm_test_server, read_litellm_test_server_request_logs+from tests.test_utils.server import read_litellm_test_server_request_logs@pytest.fixture()@@ -158,8 +158,8 @@ def test_lm_calls_skip_in_memory_cache_if_key_not_computable():def test_lms_called_expected_number_of_times_for_cache_key_generation_failures():- with pytest.raises(Exception), patch("litellm.completion") as mock_completion:- mock_completion.side_effect = Exception("Mocked exception")+ with pytest.raises(RuntimeError), patch("litellm.completion") as mock_completion:+ mock_completion.side_effect = RuntimeError("Mocked exception")lm = dspy.LM(model="openai/dspy-test-model",api_base="fakebase",diff --git a/tests/clients/test_cache.py b/tests/clients/test_cache.pyindex 5bd87abba..edff1485d 100644--- a/tests/clients/test_cache.py+++ b/tests/clients/test_cache.py@@ -1,5 +1,4 @@-import os-import shutil+from dataclasses import dataclassfrom unittest.mock import patchimport pydantic@@ -8,7 +7,6 @@ from cachetools import LRUCachefrom diskcache import FanoutCachefrom dspy.clients.cache import Cache-from dataclasses import dataclass@dataclassdiff --git a/tests/clients/test_databricks.py b/tests/clients/test_databricks.pyindex eb209e9f6..d71df8874 100644--- a/tests/clients/test_databricks.py+++ b/tests/clients/test_databricks.py@@ -4,15 +4,15 @@ This test requires valid Databricks credentials, so it is skipped on github actimanual testing."""+import pytest++import dspyfrom dspy.clients.databricks import (DatabricksProvider,- _create_directory_in_databricks_unity_catalog,TrainingJobDatabricks,+ _create_directory_in_databricks_unity_catalog,)-import pytest-import dspy-try:from databricks.sdk import WorkspaceClient@@ -66,7 +66,7 @@ def test_create_finetuning_job():job = TrainingJobDatabricks()- finetuned_model = DatabricksProvider.finetune(+ DatabricksProvider.finetune(job=job,model="meta-llama/Llama-3.2-1B",train_data=fake_training_data,diff --git a/tests/clients/test_embedding.py b/tests/clients/test_embedding.pyindex 81263b76c..52c35f2c4 100644--- a/tests/clients/test_embedding.py+++ b/tests/clients/test_embedding.py@@ -1,9 +1,10 @@-import pytest-from unittest.mock import Mock, patch+from unittest.mock import patch+import numpy as np+import pytest-from dspy.clients.embedding import Embedderimport dspy+from dspy.clients.embedding import Embedder# Mock response format similar to litellm's embedding response.diff --git a/tests/clients/test_inspect_global_history.py b/tests/clients/test_inspect_global_history.pyindex e63cfe595..2deb0c2e9 100644--- a/tests/clients/test_inspect_global_history.py+++ b/tests/clients/test_inspect_global_history.py@@ -1,23 +1,26 @@import pytest-from dspy.utils.dummies import DummyLM-from dspy.clients.base_lm import GLOBAL_HISTORY+import dspy+from dspy.clients.base_lm import GLOBAL_HISTORY+from dspy.utils.dummies import DummyLM+@pytest.fixture(autouse=True)def clear_history():GLOBAL_HISTORY.clear()yield+def test_inspect_history_basic(capsys):# Configure a DummyLM with some predefined responseslm = DummyLM([{"response": "Hello"}, {"response": "How are you?"}])dspy.settings.configure(lm=lm)-+# Make some calls to generate historypredictor = dspy.Predict("query: str -> response: str")predictor(query="Hi")predictor(query="What's up?")-+# Test inspecting all historyhistory = GLOBAL_HISTORYprint(capsys)@@ -26,45 +29,48 @@ def test_inspect_history_basic(capsys):assert all(isinstance(entry, dict) for entry in history)assert all("messages" in entry for entry in history)+def test_inspect_history_with_n(capsys):"""Test that inspect_history works with nRandom failures in this test most likely mean you are printing messages somewhere"""lm = DummyLM([{"response": "One"}, {"response": "Two"}, {"response": "Three"}])dspy.settings.configure(lm=lm)-+# Generate some historypredictor = dspy.Predict("query: str -> response: str")predictor(query="First")predictor(query="Second")predictor(query="Third")-+dspy.inspect_history(n=2)# Test getting last 2 entriesout, err = capsys.readouterr()- assert not "First" in out+ assert "First" not in outassert "Second" in outassert "Third" in out+def test_inspect_empty_history(capsys):# Configure fresh DummyLMlm = DummyLM([])dspy.settings.configure(lm=lm)-+# Test inspecting empty historydspy.inspect_history()history = GLOBAL_HISTORYassert len(history) == 0assert isinstance(history, list)+def test_inspect_history_n_larger_than_history(capsys):lm = DummyLM([{"response": "First"}, {"response": "Second"}])dspy.settings.configure(lm=lm)-+predictor = dspy.Predict("query: str -> response: str")predictor(query="Query 1")predictor(query="Query 2")-+# Request more entries than existdspy.inspect_history(n=5)history = GLOBAL_HISTORYdiff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.pyindex 3e214469c..1aa1b03aa 100644--- a/tests/clients/test_lm.py+++ b/tests/clients/test_lm.py@@ -1,16 +1,15 @@+import timefrom unittest import mockfrom unittest.mock import patch-import timeimport litellmimport pydanticimport pytest+from litellm.utils import Choices, Message, ModelResponsefrom openai import RateLimitErrorimport dspyfrom dspy.utils.usage_tracker import track_usage-from tests.test_utils.server import litellm_test_server, read_litellm_test_server_request_logs-from litellm.utils import ModelResponse, Message, Choicesdef test_chat_lms_can_be_queried(litellm_test_server):@@ -127,7 +126,10 @@ def test_lm_calls_support_callables(litellm_test_server):api_base, _ = litellm_test_serverwith mock.patch("litellm.completion", autospec=True, wraps=litellm.completion) as spy_completion:- azure_ad_token_provider = lambda *args, **kwargs: None++ def azure_ad_token_provider(*args, **kwargs):+ return None+lm_with_callable = dspy.LM(model="openai/dspy-test-model",api_base=api_base,@@ -315,12 +317,12 @@ def test_logprobs_included_when_requested():},]}- assert mock_completion.call_args.kwargs["logprobs"] == True+ assert mock_completion.call_args.kwargs["logprobs"]@pytest.mark.asyncioasync def test_async_lm_call():- from litellm.utils import ModelResponse, Message, Choices+ from litellm.utils import Choices, Message, ModelResponsemock_response = ModelResponse(choices=[Choices(message=Message(content="answer"))], model="openai/gpt-4o-mini")diff --git a/tests/conftest.py b/tests/conftest.pyindex 894534cb6..937dcb32e 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -1,9 +1,12 @@-import pytestimport copy+import pytest++from tests.test_utils.server import litellm_test_server, read_litellm_test_server_request_logs # noqa: F401SKIP_DEFAULT_FLAGS = ["reliability"]+@pytest.fixture(autouse=True)def clear_settings():"""Ensures that the settings are cleared after each test."""@@ -20,26 +23,29 @@ def clear_settings():def anyio_backend():return "asyncio"+# Taken from: https://gist.github.com/justinmklam/b2aca28cb3a6896678e2e2927c6b6a38def pytest_addoption(parser):for flag in SKIP_DEFAULT_FLAGS:parser.addoption(- "--{}".format(flag),+ f"--{flag}",action="store_true",default=False,- help="run {} tests".format(flag),+ help=f"run {flag} tests",)+def pytest_configure(config):for flag in SKIP_DEFAULT_FLAGS:config.addinivalue_line("markers", flag)+def pytest_collection_modifyitems(config, items):for flag in SKIP_DEFAULT_FLAGS:- if config.getoption("--{}".format(flag)):+ if config.getoption(f"--{flag}"):return- skip_mark = pytest.mark.skip(reason="need --{} option to run".format(flag))+ skip_mark = pytest.mark.skip(reason=f"need --{flag} option to run")for item in items:if flag in item.keywords:- item.add_marker(skip_mark)\ No newline at end of file+ item.add_marker(skip_mark)diff --git a/tests/docs/test_mkdocs_links.py b/tests/docs/test_mkdocs_links.pyindex 5d5fe7c54..0fad37ccb 100644--- a/tests/docs/test_mkdocs_links.py+++ b/tests/docs/test_mkdocs_links.py@@ -1,40 +1,38 @@---import pytestimport os+def test_nav_files_exist():# Read mkdocs.yml- docs_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'docs', 'docs')- yaml_path = os.path.join(os.path.dirname(__file__), '..', '..', 'docs', 'mkdocs.yml')-+ docs_dir = os.path.join(os.path.dirname(__file__), "..", "..", "docs", "docs")+ yaml_path = os.path.join(os.path.dirname(__file__), "..", "..", "docs", "mkdocs.yml")+# Read file and extract nav sectionwith open(yaml_path) as f:content = f.read()-+# Find nav section- nav_start = content.find('nav:')- lines = content[nav_start:].split('\n')-+ nav_start = content.find("nav:")+ lines = content[nav_start:].split("\n")+# Get markdown filesmd_files = []for line in lines:- if '.md' in line:+ if ".md" in line:# Extract the markdown filename and clean it up- md_file = line.strip().split(':')[-1].strip()+ md_file = line.strip().split(":")[-1].strip()# Remove list markers and quotes- md_file = md_file.lstrip('- ').strip("'").strip('"')- if md_file.endswith('.md'):+ md_file = md_file.lstrip("- ").strip("'").strip('"')+ if md_file.endswith(".md"):md_files.append(md_file)-+# Check if files existmissing = []for file in md_files:if not os.path.exists(os.path.join(docs_dir, file)):missing.append(file)-+print("\nChecking files in:", docs_dir)print("Found MD files:", md_files)print("Missing files:", missing)-- assert not missing, f"Missing files: {missing}"\ No newline at end of file++ assert not missing, f"Missing files: {missing}"