Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python
1.9 KB
Raw Blame
import dspy
from dspy.utils.dummies import DummyLM


class BasicQA(dspy.Signature):
    """Answer questions with short factoid answers."""

    question = dspy.InputField()
    answer = dspy.OutputField(desc="often between 1 and 5 words")


# Example completions generated by a model for reference
completions = [
    dspy.Prediction(
        rationale="I recall that during clear days, the sky often appears this color.",
        answer="blue",
    ),
    dspy.Prediction(
        rationale="Based on common knowledge, I believe the sky is typically seen as this color.",
        answer="green",
    ),
    dspy.Prediction(
        rationale="From images and depictions in media, the sky is frequently represented with this hue.",
        answer="blue",
    ),
]


def test_basic_example():
    # Pass signature to MultiChainComparison module
    compare_answers = dspy.MultiChainComparison(BasicQA)

    # Call the MultiChainComparison on the completions
    question = "What is the color of the sky?"
    lm = DummyLM([{"rationale": "my rationale", "answer": "blue"}])
    dspy.configure(lm=lm)
    final_pred = compare_answers(completions, question=question)

    assert final_pred.rationale == "my rationale"
    assert final_pred.answer == "blue"


def test_comparing_native_completions_without_summaries():
    class NativeLM(DummyLM):
        @property
        def supports_reasoning(self):
            return True

    lm = NativeLM([{"answer": "blue"}, {"answer": "green"}, {"rationale": "Choose blue", "answer": "blue"}])
    with dspy.context(lm=lm):
        candidates = dspy.ChainOfThought(BasicQA, n=2)(question="Sky color?").completions
        assert candidates.reasoning == [None, None]
        result = dspy.MultiChainComparison(BasicQA, M=2)(candidates, question="Sky color?")
    assert result.answer == "blue"
    prompt = lm.history[-1]["messages"][-1]["content"]
    assert "blue" in prompt and "green" in prompt
    assert "None" not in prompt