Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

feat(tests): fix run_tests.yml

Amir Mehr5d994d4f71f3
diff --git a/tests_integration/pytest.ini b/tests_integration/pytest.ini
index 60438fab3..250e022ee 100644
--- a/tests_integration/pytest.ini
+++ b/tests_integration/pytest.ini
@@ -3,4 +3,4 @@ filterwarnings =
ignore::DeprecationWarning
env =
- DSP_NOTEBOOK_CACHEDIR=../cache
+ DSP_NOTEBOOK_CACHEDIR=./cache
diff --git a/tests_integration/test_intro.py b/tests_integration/test_intro.py
index b00e896f3..08b657eeb 100644
--- a/tests_integration/test_intro.py
+++ b/tests_integration/test_intro.py
@@ -6,22 +6,6 @@ import pytest
import dspy
-# print("first")
-# print(os.environ.get("DSP_NOTEBOOK_CACHEDIR"))
-#
-# library_dir = Path(__file__).resolve().parent
-# base_dir = library_dir.parent
-# cache_dir = str(base_dir / "cache")
-# os.environ["DSP_NOTEBOOK_CACHEDIR"] = cache_dir
-#
-# if cache_dir and not Path(cache_dir).exists():
-# Path(cache_dir).mkdir(parents=True)
-#
-# print("second")
-#
-# print(os.environ.get("DSP_NOTEBOOK_CACHEDIR"))
-
-
class BaseIntegrationTestWithCache:
@pytest.fixture(autouse=True)
@@ -95,8 +79,8 @@ class TestIntroIntegration(BaseIntegrationTestWithCache):
assert f"Question: {my_question}" == "Question: What castle did David Gregory inherit?"
assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: Kinnairdy Castle"
assert (
- f"Retrieved Contexts (truncated): {[c[:10] + '...' for c in pred.context]}"
- == "Retrieved Contexts (truncated): ['David Greg...', 'Gregory Ta...', 'David Greg...']"
+ f"Retrieved Contexts (truncated): {[c[:10] + '...' for c in pred.context]}"
+ == "Retrieved Contexts (truncated): ['David Greg...', 'Gregory Ta...', 'David Greg...']"
)
# Verify compiled model's parameters
@@ -155,7 +139,7 @@ class TestIntroIntegration(BaseIntegrationTestWithCache):
# Assertions to verify the SimplifiedBaleen model
assert (
- f"Question: {my_question}" == "Question: How many storeys are in the castle that David Gregory inherited?"
+ f"Question: {my_question}" == "Question: How many storeys are in the castle that David Gregory inherited?"
)
assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: five"
@@ -180,7 +164,7 @@ class TestIntroIntegration(BaseIntegrationTestWithCache):
if max([len(h) for h in hops]) > 100:
return False
if any(
- dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))
+ dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))
):
return False
@@ -198,12 +182,12 @@ class TestIntroIntegration(BaseIntegrationTestWithCache):
# Assertions for the retrieval scores
assert f"## Retrieval Score for RAG: {compiled_rag_retrieval_score}" == "## Retrieval Score for RAG: 26.0"
assert (
- f"## Retrieval Score for uncompiled Baleen: {uncompiled_baleen_retrieval_score}"
- == "## Retrieval Score for uncompiled Baleen: 36.0"
+ f"## Retrieval Score for uncompiled Baleen: {uncompiled_baleen_retrieval_score}"
+ == "## Retrieval Score for uncompiled Baleen: 36.0"
)
assert (
- f"## Retrieval Score for compiled Baleen: {compiled_baleen_retrieval_score}"
- == "## Retrieval Score for compiled Baleen: 60.0"
+ f"## Retrieval Score for compiled Baleen: {compiled_baleen_retrieval_score}"
+ == "## Retrieval Score for compiled Baleen: 60.0"
)
assert compiled_baleen("How many storeys are in the castle that David Gregory inherited?") is not None
@@ -214,19 +198,19 @@ class TestIntroIntegration(BaseIntegrationTestWithCache):
# Assertions to verify the retrieval functionality
assert retrieve.k == 3
assert (
- dev_example.question
- == "What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
+ dev_example.question
+ == "What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
)
assert (
- top_k_passages[0]
- == "Restaurant: Impossible | Restaurant: Impossible is an American reality television series, featuring "
- "chef and restaurateur Robert Irvine, that aired on Food Network from 2011 to 2016."
+ top_k_passages[0]
+ == "Restaurant: Impossible | Restaurant: Impossible is an American reality television series, featuring "
+ "chef and restaurateur Robert Irvine, that aired on Food Network from 2011 to 2016."
)
assert (
- top_k_passages[1]
- == "Jean Joho | Jean Joho is a French-American chef and restaurateur. He is chef/proprietor of Everest in "
- "Chicago (founded in 1986), Paris Club Bistro & Bar and Studio Paris in Chicago, The Eiffel Tower "
- "Restaurant in Las Vegas, and Brasserie JO in Boston."
+ top_k_passages[1]
+ == "Jean Joho | Jean Joho is a French-American chef and restaurateur. He is chef/proprietor of Everest in "
+ "Chicago (founded in 1986), Paris Club Bistro & Bar and Studio Paris in Chicago, The Eiffel Tower "
+ "Restaurant in Las Vegas, and Brasserie JO in Boston."
)
assert top_k_passages[2][:30] == "List of Restaurant: Impossible"
@@ -247,8 +231,8 @@ class TestIntroIntegration(BaseIntegrationTestWithCache):
# Assertions to verify the basic QA functionality
assert (
- f"Question: {dev_example.question}"
- == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
+ f"Question: {dev_example.question}"
+ == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
)
assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: American"
@@ -259,12 +243,12 @@ class TestIntroIntegration(BaseIntegrationTestWithCache):
# Assertions to verify the chain of thought functionality
assert (
- f"Question: {dev_example.question}"
- == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
+ f"Question: {dev_example.question}"
+ == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
)
assert (
- f"Thought: {pred.rationale.split('.', 1)[1].strip()}"
- == "Thought: We know that the chef and restaurateur featured in Restaurant: Impossible is Robert Irvine."
+ f"Thought: {pred.rationale.split('.', 1)[1].strip()}"
+ == "Thought: We know that the chef and restaurateur featured in Restaurant: Impossible is Robert Irvine."
)
assert f"Predicted Answer: {pred.answer}" == "Predicted Answer: British"
@@ -281,27 +265,27 @@ class TestIntroIntegration(BaseIntegrationTestWithCache):
# Assertions to verify the dataset loading
assert (
- f"Question: {train_example.question}"
- == "Question: At My Window was released by which American singer-songwriter?"
+ f"Question: {train_example.question}"
+ == "Question: At My Window was released by which American singer-songwriter?"
)
assert f"Answer: {train_example.answer}" == "Answer: John Townes Van Zandt"
dev_example = devset[18]
assert (
- f"Question: {dev_example.question}"
- == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
+ f"Question: {dev_example.question}"
+ == "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?"
)
assert f"Answer: {dev_example.answer}" == "Answer: English"
assert "Restaurant: Impossible" in list(dev_example.gold_titles)
assert "Robert Irvine" in list(dev_example.gold_titles)
assert (
- f"For this dataset, training examples have input keys {train_example.inputs().keys()} and label keys "
- f"{train_example.labels().keys()}"
- == "For this dataset, training examples have input keys ['question'] and label keys ['answer']"
+ f"For this dataset, training examples have input keys {train_example.inputs().keys()} and label keys "
+ f"{train_example.labels().keys()}"
+ == "For this dataset, training examples have input keys ['question'] and label keys ['answer']"
)
assert (
- f"For this dataset, dev examples have input keys {dev_example.inputs().keys()} and label keys "
- f"{dev_example.labels().keys()}"
- == "For this dataset, dev examples have input keys ['question'] and label keys ['answer', 'gold_titles']"
+ f"For this dataset, dev examples have input keys {dev_example.inputs().keys()} and label keys "
+ f"{dev_example.labels().keys()}"
+ == "For this dataset, dev examples have input keys ['question'] and label keys ['answer', 'gold_titles']"
)
return dev_example, devset, trainset