DSPy: The framework for programming—not prompting—language models
Python
import dspy
from dspy.utils.dummies import DummyLM
class BasicQA(dspy.Signature):
"""Answer questions with short factoid answers."""
question = dspy.InputField()
answer = dspy.OutputField(desc="often between 1 and 5 words")
# Example completions generated by a model for reference
completions = [
dspy.Prediction(
rationale="I recall that during clear days, the sky often appears this color.",
answer="blue",
),
dspy.Prediction(
rationale="Based on common knowledge, I believe the sky is typically seen as this color.",
answer="green",
),
dspy.Prediction(
rationale="From images and depictions in media, the sky is frequently represented with this hue.",
answer="blue",
),
]
def test_basic_example():
# Pass signature to MultiChainComparison module
compare_answers = dspy.MultiChainComparison(BasicQA)
# Call the MultiChainComparison on the completions
question = "What is the color of the sky?"
lm = DummyLM([{"rationale": "my rationale", "answer": "blue"}])
dspy.configure(lm=lm)
final_pred = compare_answers(completions, question=question)
assert final_pred.rationale == "my rationale"
assert final_pred.answer == "blue"
def test_comparing_native_completions_without_summaries():
class NativeLM(DummyLM):
@property
def supports_reasoning(self):
return True
lm = NativeLM([{"answer": "blue"}, {"answer": "green"}, {"rationale": "Choose blue", "answer": "blue"}])
with dspy.context(lm=lm):
candidates = dspy.ChainOfThought(BasicQA, n=2)(question="Sky color?").completions
assert candidates.reasoning == [None, None]
result = dspy.MultiChainComparison(BasicQA, M=2)(candidates, question="Sky color?")
assert result.answer == "blue"
prompt = lm.history[-1]["messages"][-1]["content"]
assert "blue" in prompt and "green" in prompt
assert "None" not in prompt