Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

feat(dspy): add docs

JONEMI19f3db229496b8
diff --git a/docs/docs/deep-dive/signature/internal-signatures.mdx b/docs/docs/deep-dive/signature/internal-signatures.mdx
new file mode 100644
index 000000000..f971ad922
--- /dev/null
+++ b/docs/docs/deep-dive/signature/internal-signatures.mdx
@@ -0,0 +1,85 @@
+---
+sidebar_position: 3
+---
+
+import AuthorDetails from "@site/src/components/AuthorDetails";
+
+## DSPy's Internal Signatures
+
+DSPy makes extensive use of signatures internally to define tasks and their requirements. Signatures are used to define the inputs and outputs of a task, and optionally, a small description about them and the task too.
+
+Sometimes, you may want to update the signature of a task to include instructions or additional fields. You can access and replace any signature in DSPy using the `dspy.Signature.replace` context manager.
+
+## Updating Signatures
+
+To update a signature, you can use the `dspy.Signature.replace` context manager. This context manager allows you to replace the signature of a task with a new one. Here's an example of updating one of the signatures used in the Copro optimizer, to add additional instructions:
+
+```python
+from dspy.teleprompt import copro_optimizer
+import dspy
+
+class MyBasicGenerateInstruction(mipro_optimizer.BasicGenerateInstruction):
+ """
+ <PERSONA>
+ You are an instruction optimizer for large language models.
+ </PERSONA>
+ <TASK>
+ I will give you a ``signature`` of fields (inputs and outputs) in English. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative, but the new instruction you propose should be clear and concise.
+ </TASK>
+ """
+ basic_instruction = dspy.InputField(desc="The unoptimized instruction. New instructions should achieve the same goals.")
+
+with mipro_optimizer.BasicGenerateInstruction.replace(MyBasicGenerateInstruction):
+ teleprompter = COPRO(prompt_model=prompt_model, metric=metric, breadth=BREADTH, depth=DEPTH, init_temperature=INIT_TEMPERATURE)
+ kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)
+ compiled_prompt_opt = teleprompter.compile(program.deepcopy(), trainset=trainset[:DEV_NUM], eval_kwargs=kwargs)
+ eval_score = evaluate(compiled_prompt_opt, devset=evalset[:EVAL_NUM], **kwargs)
+
+```
+
+Which will now update the signature of the `BasicGenerateInstruction` task in the Copro optimizer to include the new `basic_instruction` field.
+
+You can replace any signature in DSPy using this method, and it will be used so long as the context manager is active.
+
+There is also a helper function `dspy.update_signatures` that can be used to update multiple signatures at once. This function takes a dictionary of signatures to update, with the key being the signature to update and the value being the new signature class.
+
+```python
+
+from dspy.teleprompt import copro_optimizer
+import dspy
+
+class MyBasicGenerateInstruction(mipro_optimizer.BasicGenerateInstruction):
+ """
+ <PERSONA>
+ You are an instruction optimizer for large language models.
+ </PERSONA>
+ <TASK>
+ I will give you a ``signature`` of fields (inputs and outputs) in English. Your task is to propose an instruction that will lead a good language model to perform the task well. Don't be afraid to be creative, but the new instruction you propose should be clear and concise.
+ </TASK>
+ """
+ basic_instruction = dspy.InputField(desc="The unoptimized instruction. New instructions should achieve the same goals.")
+
+class GenerateInstructionGivenAttempts(mipro_optimizer.GenerateInstructionGivenAttempts):
+ """
+ <PERSONA>
+ You are an instruction optimizer for large language models.
+ </PERSONA>
+ <TASK>
+ I will give some task instructions I've tried, along with their corresponding validation scores. The instructions are arranged in increasing order based on their scores, where higher scores indicate better quality.
+
+ Your task is to propose a new instruction that will lead a good language model to perform the task even better. Don't be afraid to be creative
+ </TASK>
+ """
+ pass
+
+with dspy.update_signatures({
+ copro_optimizer.BasicGenerateInstruction: MyBasicGenerateInstruction,
+ copro_optimizer.GenerateInstructionGivenAttempts: GenerateInstructionGivenAttempts
+ }):
+ ...
+
+```
+
+---
+
+<AuthorDetails name="Michael Jones" />
diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.py
index fce3c2a2a..e8e9c4014 100644
--- a/dspy/signatures/signature.py
+++ b/dspy/signatures/signature.py
@@ -201,7 +201,6 @@ class SignatureMeta(type(BaseModel)):
#
# For compatibility with the legacy dsp format, you can use the signature_to_template function.
#
-T_co = typing.TypeVar("T_co", bound="Signature", covariant=True)
class Signature(BaseModel, metaclass=SignatureMeta):
@@ -214,8 +213,8 @@ class Signature(BaseModel, metaclass=SignatureMeta):
@classmethod
@contextmanager
def replace(
- cls: T_co,
- new_signature: T_co,
+ cls: "Signature",
+ new_signature: "Signature",
validate_new_signiture: bool = True,
) -> typing.Generator[None, None, None]:
"""Replace the signature with an updated version.
@@ -234,12 +233,20 @@ class Signature(BaseModel, metaclass=SignatureMeta):
f"Field '{field}' is missing from the updated signiture '{new_signature.__class__}.",
)
- old_signature = cls
- setattr(inspect.getmodule(cls), cls.__name__, new_signature)
+ class OldSignature(cls, Signature):
+ pass
+
+ replace_fields = ["__doc__", "model_fields", "model_extra", "model_config"]
+ for field in replace_fields:
+ setattr(cls, field, getattr(new_signature, field))
+ cls.model_rebuild(force=True)
yield
- setattr(inspect.getmodule(cls), cls.__name__, old_signature)
+ for field in replace_fields:
+ setattr(cls, field, getattr(OldSignature, field))
+ cls.model_rebuild(force=True)
+@contextmanager
def update_signatures(
signature_map: Dict[Type[Signature], Type[Signature]],
validate_new_signiture: bool = True,
diff --git a/tests/signatures/test_signature.py b/tests/signatures/test_signature.py
index e635455b9..78501d20d 100644
--- a/tests/signatures/test_signature.py
+++ b/tests/signatures/test_signature.py
@@ -1,10 +1,11 @@
import textwrap
-import pytest
-import pydantic
-from dspy import Signature, infer_prefix, InputField, OutputField
from typing import List
+import pydantic
+import pytest
+
import dspy
+from dspy import InputField, OutputField, Signature, infer_prefix
from dspy.utils.dummies import DummyLM
@@ -194,17 +195,63 @@ def test_multiline_instructions():
dspy.settings.configure(lm=lm)
assert predictor().output == "short answer"
- assert lm.get_convo(-1) == textwrap.dedent("""\
+ assert lm.get_convo(-1) == textwrap.dedent(
+ """\
First line
Second line
Third line
-
+
---
-
+
Follow the following format.
-
+
Output: ${output}
-
+
---
-
- Output: short answer""")
+
+ Output: short answer"""
+ )
+
+
+def test_replaced_by_replace_context_manager():
+ class SignatureOne(Signature):
+ input1 = InputField()
+ output = OutputField()
+
+ class SignatureTwo(Signature):
+ input2 = InputField()
+ output = OutputField()
+
+ with SignatureOne.replace(SignatureTwo, validate_new_signiture=False):
+ # assert SignatureOne.input_fields has been replaced with SignatureTwo.input_fields
+ assert "input2" in SignatureOne.input_fields
+ # after the context manager, the original input_fields should be restored
+ assert SignatureOne.input_fields["input1"].json_schema_extra["prefix"] == "Input 1:"
+
+
+def test_multiple_replaced_by_update_signatures():
+ class SignatureOne(Signature):
+ input1 = InputField()
+ output = OutputField()
+
+ class SignatureTwo(Signature):
+ input2 = InputField()
+ output = OutputField()
+
+ class SignatureThree(Signature):
+ input3 = InputField()
+ output = OutputField()
+
+ class SignatureFour(Signature):
+ input4 = InputField()
+ output = OutputField()
+
+ signature_map = {
+ SignatureOne: SignatureThree,
+ SignatureTwo: SignatureFour,
+ }
+ with dspy.update_signatures(signature_map, validate_new_signiture=False):
+ assert "input3" in SignatureOne.input_fields
+ assert "input4" in SignatureTwo.input_fields
+ assert "input1" in SignatureOne.input_fields
+ assert "input2" in SignatureTwo.input_fields