Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

First try at a new optimizer/teleprompt

Thomas D Ahle7980a2bbd3df
diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py
index da5e52f94..3d6c47141 100644
--- a/dspy/functional/functional.py
+++ b/dspy/functional/functional.py
@@ -1,4 +1,3 @@
-from collections import defaultdict
import inspect
import os
import openai
@@ -14,7 +13,8 @@ from dspy.primitives.prediction import Prediction
from dspy.signatures.signature import ensure_signature, make_signature
-MAX_RETRIES = 3
+# Some improvement ideas:
+# - Increase the temperature on error
def predictor(func) -> dspy.Module:
@@ -56,7 +56,7 @@ class FunctionalModule(dspy.Module):
self.__dict__[name] = attr.copy()
-def TypedChainOfThought(signature) -> dspy.Module: # noqa: N802
+def TypedChainOfThought(signature, max_retries=3) -> dspy.Module: # noqa: N802
"""Just like TypedPredictor, but adds a ChainOfThought OutputField."""
signature = ensure_signature(signature)
output_keys = ", ".join(signature.output_fields.keys())
@@ -68,17 +68,19 @@ def TypedChainOfThought(signature) -> dspy.Module: # noqa: N802
desc="${produce the " + output_keys + "}. We ...",
),
),
+ max_retries=max_retries,
)
class TypedPredictor(dspy.Module):
- def __init__(self, signature):
+ def __init__(self, signature, max_retries=3):
super().__init__()
self.signature = ensure_signature(signature)
self.predictor = dspy.Predict(signature)
+ self.max_retries = max_retries
def copy(self) -> "TypedPredictor":
- return TypedPredictor(self.signature)
+ return TypedPredictor(self.signature, self.max_retries)
@staticmethod
def _make_example(type_) -> str:
@@ -140,6 +142,15 @@ class TypedPredictor(dspy.Module):
format_ = passages2text
elif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):
format_ = lambda x: x if isinstance(x, str) else x.model_dump_json()
+ # Special formatting for lists of known types. Maybe the output fields sohuld have this too?
+ elif typing.get_origin(type_) in (List, list, Tuple, tuple):
+ (inner_type,) = typing.get_args(type_)
+ if inspect.isclass(inner_type) and issubclass(inner_type, pydantic.BaseModel):
+ format_ = (
+ lambda x: x if isinstance(x, str) else "[" + ",".join(i.model_dump_json() for i in x) + "]"
+ )
+ else:
+ format_ = lambda x: x if isinstance(x, str) else json.dumps(x)
signature = signature.with_updated_fields(name, format=format_)
return signature
@@ -149,40 +160,46 @@ class TypedPredictor(dspy.Module):
# We have to re-prepare the signature on every forward call, because the base
# signature might have been modified by an optimizer or something like that.
signature = self._prepare_signature()
- for try_i in range(MAX_RETRIES):
+ for try_i in range(self.max_retries):
result = self.predictor(**modified_kwargs, new_signature=signature)
errors = {}
parsed_results = []
# Parse the outputs
- for i, completion in enumerate(result.completions):
- try:
- parsed = {}
- for name, field in signature.output_fields.items():
+ for completion in result.completions:
+ parsed = {}
+ for name, field in signature.output_fields.items():
+ try:
value = completion[name]
parser = field.json_schema_extra.get("parser", lambda x: x)
parsed[name] = parser(value)
- # Instantiate the actual signature with the parsed values.
- # This allow pydantic to validate the fields defined in the signature.
+ except (pydantic.ValidationError, ValueError) as e:
+ errors[name] = _format_error(e)
+ # If we can, we add an example to the error message
+ current_desc = field.json_schema_extra.get("desc", "")
+ i = current_desc.find("JSON Schema: ")
+ if i == -1:
+ continue # Only add examples to JSON objects
+ suffix, current_desc = current_desc[i:], current_desc[:i]
+ prefix = "You MUST use this format: "
+ if (
+ try_i + 1 < self.max_retries
+ and prefix not in current_desc
+ and (example := self._make_example(field.annotation))
+ ):
+ signature = signature.with_updated_fields(
+ name,
+ desc=current_desc + "\n" + prefix + example + "\n" + suffix,
+ )
+ # No reason trying to parse the general signature, or run more completions, if we already have errors
+ if errors:
+ break
+ # Instantiate the actual signature with the parsed values.
+ # This allow pydantic to validate the fields defined in the signature.
+ try:
_dummy = self.signature(**kwargs, **parsed)
parsed_results.append(parsed)
- except (pydantic.ValidationError, ValueError) as e:
- errors[name] = _format_error(e)
- # If we can, we add an example to the error message
- current_desc = field.json_schema_extra.get("desc", "")
- i = current_desc.find("JSON Schema: ")
- if i == -1:
- continue # Only add examples to JSON objects
- suffix, current_desc = current_desc[i:], current_desc[:i]
- prefix = "You MUST use this format: "
- if (
- try_i + 1 < MAX_RETRIES
- and prefix not in current_desc
- and (example := self._make_example(field.annotation))
- ):
- signature = signature.with_updated_fields(
- name,
- desc=current_desc + "\n" + prefix + example + "\n" + suffix,
- )
+ except pydantic.ValidationError as e:
+ errors["general"] = _format_error(e)
if errors:
# Add new fields for each error
for name, error in errors.items():
@@ -209,7 +226,7 @@ def _format_error(error: Exception):
if isinstance(error, pydantic.ValidationError):
errors = []
for e in error.errors():
- fields = ", ".join(e["loc"])
+ fields = ", ".join(map(str, e["loc"]))
errors.append(f"{e['msg']}: {fields} (error type: {e['type']})")
return "; ".join(errors)
else:
diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py
index 60e668a83..d0a5f6010 100644
--- a/dspy/predict/predict.py
+++ b/dspy/predict/predict.py
@@ -6,6 +6,7 @@ from dspy.primitives.prediction import Prediction
from dspy.signatures.signature import ensure_signature, signature_to_template
+
class Predict(Parameter):
def __init__(self, signature, **config):
self.stage = random.randbytes(8).hex()
@@ -27,7 +28,7 @@ class Predict(Parameter):
state["signature_instructions"] = self.signature.instructions
*_, last_key = self.signature.fields.keys()
- state["signature_prefix"] = self.signature.fields[last_key].json_schema_extra['prefix']
+ state["signature_prefix"] = self.signature.fields[last_key].json_schema_extra["prefix"]
return state
@@ -85,6 +86,8 @@ class Predict(Parameter):
# Switch to legacy format for dsp.generate
template = signature_to_template(signature)
+ # print("Created template", template)
+ # print("From Signature", signature)
if self.lm is None:
x, C = dsp.generate(template, **config)(x, stage=self.stage)
@@ -103,7 +106,8 @@ class Predict(Parameter):
for field in template.fields:
if field.output_variable not in kwargs.keys():
completions[-1][field.output_variable] = getattr(
- c, field.output_variable,
+ c,
+ field.output_variable,
)
pred = Prediction.from_completions(completions, signature=signature)
diff --git a/dspy/primitives/module.py b/dspy/primitives/module.py
index 1b4a342f7..59ec8a09f 100644
--- a/dspy/primitives/module.py
+++ b/dspy/primitives/module.py
@@ -8,7 +8,7 @@ class BaseModule:
def named_parameters(self):
"""
- Unlike PyTorch, handles (non-recursive) lists of parameters too.
+ Unlike PyTorch, handles (non-recursive) lists of parameters too.
"""
from dspy.predict.parameter import Parameter
@@ -27,10 +27,10 @@ class BaseModule:
elif isinstance(value, BaseModule):
# When a sub-module is pre-compiled, keep it frozen.
- if not getattr(value, '_compiled', False):
+ if not getattr(value, "_compiled", False):
for sub_name, param in value.named_parameters():
add_parameter(f"{name}.{sub_name}", param)
-
+
elif isinstance(value, (list, tuple)):
for idx, item in enumerate(value):
add_parameter(f"{name}[{idx}]", item)
@@ -49,23 +49,23 @@ class BaseModule:
def reset_copy(self):
obj = copy.deepcopy(self)
-
+
for param in obj.parameters():
param.reset()
-
+
return obj
-
+
def dump_state(self):
return {name: param.dump_state() for name, param in self.named_parameters()}
-
+
def load_state(self, state):
for name, param in self.named_parameters():
param.load_state(state[name])
-
+
def save(self, path):
with open(path, "w") as f:
f.write(ujson.dumps(self.dump_state(), indent=2))
-
+
def load(self, path):
with open(path) as f:
self.load_state(ujson.loads(f.read()))
diff --git a/dspy/primitives/program.py b/dspy/primitives/program.py
index 5e063bfb0..512e7ce67 100644
--- a/dspy/primitives/program.py
+++ b/dspy/primitives/program.py
@@ -1,4 +1,3 @@
-
from dspy.primitives.module import BaseModule
from dspy.primitives.assertions import *
import re
@@ -16,7 +15,6 @@ class ProgramMeta(type):
class Module(BaseModule, metaclass=ProgramMeta):
-
def _base_init(self):
self._compiled = False
@@ -29,12 +27,7 @@ class Module(BaseModule, metaclass=ProgramMeta):
def named_predictors(self):
from dspy.predict.predict import Predict
- named_parameters = self.named_parameters()
- return [
- (name, param)
- for name, param in named_parameters
- if isinstance(param, Predict)
- ]
+ return [(name, param) for name, param in self.named_parameters() if isinstance(param, Predict)]
def predictors(self):
return [param for _, param in self.named_predictors()]
@@ -52,7 +45,7 @@ class Module(BaseModule, metaclass=ProgramMeta):
for name, predictor in self.named_predictors():
set_attribute_by_name(self, name, func(predictor))
return self
-
+
def activate_assertions(self, handler=backtrack_handler, **handler_args):
"""
Activates assertions for the module.
diff --git a/dspy/teleprompt/signature_opt2.py b/dspy/teleprompt/signature_opt2.py
new file mode 100644
index 000000000..1a9045ae2
--- /dev/null
+++ b/dspy/teleprompt/signature_opt2.py
@@ -0,0 +1,248 @@
+import random
+from typing import Generic, Literal, TypeVar, Type
+
+import pydantic
+import dspy
+from dspy.functional.functional import TypedChainOfThought
+from dspy.signatures import Signature
+from dspy import BaseModel
+
+# TODO: Consider using the prompt optimizer to optimize the prompt optimizer :O
+
+
+def make_info(signature: type[Signature]) -> BaseModel:
+ """Creates a SignatureInfo pydantic type, that describes the Signature.
+
+ Returns an instnce of this type, with the instructions and field descriptions of the input type.
+ """
+ # First, create the SignatureInfo type
+ fields = {
+ "instructions": (str, pydantic.Field(description="The instructions for the task")),
+ }
+ for name in signature.fields:
+ fields[name + "_prefix"] = (str, pydantic.Field(description=f"The prefix for {name}"))
+ fields[name + "_desc"] = (str, pydantic.Field(description=f"The description for {name}"))
+ SignatureInfo = pydantic.create_model( # noqa: N806
+ f"SignatureInfo[{signature.__name__}]",
+ **fields,
+ )
+
+ # Add a method to convert the SignatureInfo back into a Signature
+ def to_signature(info):
+ new_signature = signature.with_instructions(info.instructions)
+ for name in signature.fields:
+ new_signature = new_signature.with_updated_fields(
+ name,
+ prefix=getattr(info, name + "_prefix"),
+ desc=getattr(info, name + "_desc"),
+ )
+ return new_signature
+
+ SignatureInfo.to_signature = to_signature
+
+ # Finally, make an instance of the SignatureInfo type with the signature's
+ # default instructions and field descriptions
+ values = {"instructions": signature.instructions}
+ for name, field in signature.fields.items():
+ values[name + "_prefix"] = field.json_schema_extra["prefix"]
+ values[name + "_desc"] = field.json_schema_extra["desc"]
+ return SignatureInfo(**values)
+
+
+T = TypeVar("T", bound=BaseModel)
+
+
+# Note: This function wouldn't be necessary if we could make the number of prompts a generic parameter of the class,
+# but alas it seems like this isn't possible in Python right now. The main reason being that types and generics only
+# live inside the type system, and can't be used to generate code at runtime.
+def make_initial_signature(n_prompts: int) -> Type[Signature]:
+ """Creates a GenerateInstructionInitial signature with the given number of initial prompts."""
+
+ class GenerateInstructionInitial(Signature, Generic[T]):
+ """You are a creative instruction optimizer for large language models.
+
+ I will give you a ``signature`` of fields (inputs and outputs) in English.
+ Your task is to propose variations of the signature that will lead a good language model.
+
+ Be very creative and think out of the box. Consider using inspiration such as:
+ Openers:
+ # You are as smart as ChatGPT.
+ # You are highly intelligent.
+ # You are an expert mathematician.
+ # You are a professor of mathematics.
+ Task Descriptions:
+ # Solve the following math question.
+ # Answer the following math question.
+ Closers:
+ # This will be fun!
+ # Take a deep breath and think carefully.
+ # I really need your help!
+ """
+
+ basic_signature: T = dspy.InputField()
+ proposed_signatures: list[T] = dspy.OutputField(
+ desc=f"A list of {n_prompts} very different variations of the basic signature",
+ min_items=n_prompts,
+ max_items=n_prompts,
+ )
+
+ return GenerateInstructionInitial
+
+
+class ScoredSignature(BaseModel, Generic[T]):
+ signature: T
+ score: float = dspy.Field(gt=0, lt=100)
+
+
+class GenerateInstructionGivenAttempts(dspy.Signature, Generic[T]):
+ """You are an instruction optimizer for large language models.
+
+ I will give some task instructions I've tried, along with their corresponding validation scores.
+ - The instructions are arranged in increasing order based on their scores, where higher scores indicate better quality.
+ - Your task is to propose a new instruction that will lead a good language model to perform the task even better.
+ - Be creative, and think out of the box.
+ - Don't repeat instructions, descriptions and prefixes that have already been attempted.
+ """
+
+ attempted_signatures: list[ScoredSignature[T]] = dspy.InputField()
+ proposed_signature: T = dspy.OutputField(desc="Next signature to try")
+ # expected_score: float = dspy.OutputField(desc="The expected score for the new signature")
+
+
+def optimize_signature(
+ student,
+ evaluator,
+ n_iterations=10,
+ strategy: Literal["best", "last"] = "best",
+ # Formerly part of the constructor
+ prompt_model=None,
+ initial_prompts=2,
+ temperature=1.4,
+ verbose=False,
+) -> dspy.Program:
+ """Create a new program that is optimized for the given task.
+
+ `student` is a program that needs to be optimized,
+ note that it may be zero-shot or already pre-optimized for demos != [].
+
+ Parameters
+ ----------
+ student : dspy.Program
+ The program to optimize.
+ evaluator : dspy.Evaluator
+ The evaluator to use to score the program.
+ n_iterations : int, optional
+ The number of iterations to run, by default 10
+ strategy : Literal["best", "last"], optional
+ The strategy to use to select the final program, by default "best"
+ prompt_model : dspy.LanguageModel, optional
+ The language model to use to generate prompts, by default None
+ initial_prompts : int, optional
+ The number of initial prompts to generate, by default 2.
+ Note that we also use the "plain" signature as a prompt, so the total number of prompts is initial_prompts + 1.
+ temperature : float, optional
+ The temperature to use when generating new prompts, by default 1.4
+ verbose : bool, optional
+ Whether to print debug information, by default False
+ """
+ prompt_model = prompt_model or dspy.settings.lm
+ MyGenerateInstructionInitial = make_initial_signature(initial_prompts) # noqa: N806
+
+ module = student.deepcopy()
+ # For some reason named_predictors sometimes returns an empty list, so we use named_parameters instead
+ named_predictors = module.named_parameters()
+ if verbose:
+ print("All predictors:")
+ print(f"{named_predictors=}")
+
+ candidates = {}
+ scores = []
+
+ # First round, just use initial prompts
+ for name, p in named_predictors:
+ candidates[name] = [make_info(p.signature)]
+
+ # Make some initial candidates
+ with dspy.settings.context(lm=prompt_model):
+ # TODO: Parallelize this
+ for name, p in named_predictors:
+ if verbose:
+ print(f"Generating new signature for {p}...")
+ info = candidates[name][0] # Use initial info, to make sure types are identical
+ generator = TypedChainOfThought(MyGenerateInstructionInitial[type(info)])
+ candidates[name] += generator(
+ basic_signature=info,
+ config={"temperature": temperature},
+ ).proposed_signatures
+ assert len(candidates[name]) == initial_prompts + 1 # Basic signature + initial prompts
+
+ candidates[name] = [
+ info.model_copy(update={"instructions": info.instructions + f"({i})"})
+ for i, info in enumerate(candidates[name])
+ ]
+
+ for i, c in enumerate(candidates[name]):
+ print(f"Generated candidate {i}:")
+ print(c.to_signature())
+
+ # Main loop of scoring + generating new candidates
+ for i in range(n_iterations):
+ if verbose:
+ print("\n" + "=" * 80)
+ print(f"Running eval iteration {i}...")
+
+ # Test candidate i
+ for p in module.predictors():
+ print(f"Installing signature {i}: ")
+ print(candidates[name][i].to_signature())
+ p.signature = candidates[name][i].to_signature()
+
+ score = evaluator(module)
+ score += random.random() * 10
+ scores.append(score)
+
+ if verbose:
+ print(f"Scores for iteration {i}: {score}")
+
+ # If we are still testing initial prompts, continue
+ if i + 1 < len(next(iter(candidates.values()))):
+ continue
+
+ # If we are done, there's no need to generate new candidates
+ if i + 1 == n_iterations:
+ break
+
+ # Otherwise generate the next candidate
+ with dspy.settings.context(lm=prompt_model):
+ # TODO: Parallelize this
+ for name, p in named_predictors:
+ SignatureInfo = type(candidates[name][0]) # noqa: N806
+ generator = TypedChainOfThought(GenerateInstructionGivenAttempts[SignatureInfo])
+ attempted_signatures = [
+ ScoredSignature[SignatureInfo](signature=info, score=sc)
+ for info, sc in zip(candidates[name], scores)
+ ]
+ attempted_signatures.sort(key=lambda x: x.score)
+ if verbose:
+ print(
+ f"Generating new signature for {name} based on {len(attempted_signatures)} previous signatures..."
+ )
+ new_signature = generator(
+ attempted_signatures=attempted_signatures,
+ config={"temperature": temperature},
+ ).proposed_signature
+ if verbose:
+ print("Generated candidate:")
+ print(new_signature.to_signature())
+ candidates[name].append(new_signature)
+
+ if strategy == "last":
+ return module
+
+ if strategy == "best":
+ i = scores.index(max(scores))
+ for name, p in named_predictors:
+ p.signature = candidates[name][i].to_signature()
+ return module
+
+ raise ValueError(f"Invalid strategy: {strategy}")
diff --git a/examples/generation.py b/examples/generation.py
new file mode 100644
index 000000000..36d3d1c92
--- /dev/null
+++ b/examples/generation.py
@@ -0,0 +1,28 @@
+from pydantic import BaseModel, Field
+from dspy.teleprompt import LabeledFewShot
+from dspy.functional import TypedPredictor
+
+import dspy
+turbo = dspy.OpenAI(model='gpt-3.5-turbo')
+colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')
+dspy.settings.configure(lm=turbo, rm=colbertv2_wiki17_abstracts)
+
+class SyntheticFact(BaseModel):
+ fact: str = Field(..., description="a statement")
+ varacity: bool = Field(..., description="is the statement true or false")
+
+class ExampleSignature(dspy.Signature):
+ """Generate an example of a synthetic fact."""
+ fact: SyntheticFact = dspy.OutputField()
+
+generator = TypedPredictor(ExampleSignature)
+examples = generator(config=dict(n=10))
+
+# If you have examples and want more
+existing_examples = [
+ dspy.Example(fact="The sky is blue", varacity=True),
+ dspy.Example(fact="The sky is green", varacity=False),
+]
+trained = LabeledFewShot().compile(student=generator, trainset=existing_examples)
+
+augmented_examples = trained(config=dict(n=10))
diff --git a/examples/nli/scone/ScoNe b/examples/nli/scone/ScoNe
new file mode 160000
index 000000000..b02532a2f
--- /dev/null
+++ b/examples/nli/scone/ScoNe
@@ -0,0 +1 @@
+Subproject commit b02532a2f4185c6118a57a148455e0750592d8c8
diff --git a/examples/quiz/DSPy_QuizGen_Cache b/examples/quiz/DSPy_QuizGen_Cache
new file mode 160000
index 000000000..27d6d433e
--- /dev/null
+++ b/examples/quiz/DSPy_QuizGen_Cache
@@ -0,0 +1 @@
+Subproject commit 27d6d433e73b91d3cf677ecf1d757813fcbd611d
diff --git a/examples/signature_opt2.ipynb b/examples/signature_opt2.ipynb
new file mode 100644
index 000000000..9edc0d4c4
--- /dev/null
+++ b/examples/signature_opt2.ipynb
@@ -0,0 +1,1717 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "The autoreload extension is already loaded. To reload it, use:\n",
+ " %reload_ext autoreload\n"
+ ]
+ }
+ ],
+ "source": [
+ "%load_ext autoreload\n",
+ "%autoreload 2\n",
+ "import dspy\n",
+ "import os\n",
+ "os.environ['OPENAI_API_KEY'] = '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'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "turbo = dspy.OpenAI(model='gpt-3.5-turbo', max_tokens=4000)\n",
+ "gpt4 = dspy.OpenAI(model='gpt-4', max_tokens=4000)\n",
+ "dspy.settings.configure(lm=turbo)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(20, 50)"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from dspy.datasets import HotPotQA\n",
+ "\n",
+ "# Load the dataset.\n",
+ "dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)\n",
+ "\n",
+ "# Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.\n",
+ "trainset = [x.with_inputs('question') for x in dataset.train]\n",
+ "devset = [x.with_inputs('question') for x in dataset.dev]\n",
+ "\n",
+ "len(trainset), len(devset)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class BasicQA(dspy.Signature):\n",
+ " \"\"\"Answer questions with short factoid answers.\"\"\"\n",
+ "\n",
+ " question = dspy.InputField()\n",
+ " answer = dspy.OutputField(desc=\"often between 1 and 5 words\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "All predictors:\n",
+ "named_predictors=[('predictor', Predict(BasicQA(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:'})\n",
+ ")))]\n",
+ "Generating new signature for Predict(BasicQA(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:'})\n",
+ "))...\n",
+ "Generated candidate 0:\n",
+ "StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.(0)'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:'})\n",
+ ")\n",
+ "Generated candidate 1:\n",
+ "StringSignature(question -> answer\n",
+ " instructions='Answer series of questions where answers must share a common theme.(1)'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Q1,', 'desc': '${question}'})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'charge amplifier designed based by Moog on stripline technology', '__dspy_field_type': 'output', 'prefix': 'A1 Wolfgang Amplifier,'})\n",
+ ")\n",
+ "Generated candidate 2:\n",
+ "StringSignature(question -> answer\n",
+ " instructions='Simulate exploratory dialogue between two people one Questioner and other Provider, Respond to each Q- below.(2)'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Q-What Computability study ranges?', 'desc': 'may also involve strings produced by non-deterministic computation'})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'answer DescHash Pure stainless Aberdeen stimulating Victoria names central whiskey article promise twitch Ohio Amber how statements board!', '__dspy_field_type': 'output', 'prefix': 'A-The explain phase-leading out question onset sound crawled fundingProblem in such separated syllabled band phrase assist reduction Haus mutual widse phoneme runtime shruproperholderstoInt valuepirizeThunder'})\n",
+ ")\n",
+ "\n",
+ "================================================================================\n",
+ "Running eval iteration 0...\n",
+ "Installing signature 0: \n",
+ "StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.(0)'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:'})\n",
+ ")\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 5 / 20 (25.0): 100%|██████████| 20/20 [00:00<00:00, 5144.18it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 5 / 20 (25.0%)\n",
+ "Scores for iteration 0: 28.55243378134378\n",
+ "\n",
+ "================================================================================\n",
+ "Running eval iteration 1...\n",
+ "Installing signature 1: \n",
+ "StringSignature(question -> answer\n",
+ " instructions='Answer series of questions where answers must share a common theme.(1)'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Q1,', 'desc': '${question}'})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'charge amplifier designed based by Moog on stripline technology', '__dspy_field_type': 'output', 'prefix': 'A1 Wolfgang Amplifier,'})\n",
+ ")\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 5 / 20 (25.0): 100%|██████████| 20/20 [00:00<00:00, 4207.98it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 5 / 20 (25.0%)\n",
+ "Scores for iteration 1: 29.911449495835697\n",
+ "\n",
+ "================================================================================\n",
+ "Running eval iteration 2...\n",
+ "Installing signature 2: \n",
+ "StringSignature(question -> answer\n",
+ " instructions='Simulate exploratory dialogue between two people one Questioner and other Provider, Respond to each Q- below.(2)'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Q-What Computability study ranges?', 'desc': 'may also involve strings produced by non-deterministic computation'})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'answer DescHash Pure stainless Aberdeen stimulating Victoria names central whiskey article promise twitch Ohio Amber how statements board!', '__dspy_field_type': 'output', 'prefix': 'A-The explain phase-leading out question onset sound crawled fundingProblem in such separated syllabled band phrase assist reduction Haus mutual widse phoneme runtime shruproperholderstoInt valuepirizeThunder'})\n",
+ ")\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 5 / 20 (25.0): 100%|██████████| 20/20 [00:00<00:00, 4577.93it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 5 / 20 (25.0%)\n",
+ "Scores for iteration 2: 34.58455944537296\n",
+ "Generating new signature for predictor based on 3 previous signatures...\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "from dspy.evaluate import Evaluate\n",
+ "from dspy.evaluate.metrics import answer_exact_match\n",
+ "from dspy.functional import TypedPredictor\n",
+ "from dspy.teleprompt.signature_opt2 import optimize_signature\n",
+ "\n",
+ "evaluator = Evaluate(devset=devset, metric=answer_exact_match, num_threads=10, display_progress=True)\n",
+ "\n",
+ "program = optimize_signature(\n",
+ " student=TypedPredictor(BasicQA),\n",
+ " evaluator=Evaluate(devset=trainset, metric=answer_exact_match, num_threads=10, display_progress=True),\n",
+ " initial_prompts=2,\n",
+ " n_iterations=8,\n",
+ " verbose=True,\n",
+ " prompt_model=gpt4,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "Given the fields `attempted_signatures`, produce the fields `proposed_signature`.\n",
+ "\n",
+ "---\n",
+ "\n",
+ "Follow the following format.\n",
+ "\n",
+ "Attempted Signatures: ${attempted_signatures}\n",
+ "Reasoning: Let's think step by step in order to ${produce the proposed_signature}. We ...\n",
+ "Proposed Signature: The improved signature for the language model. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n",
+ "\n",
+ "---\n",
+ "\n",
+ "Attempted Signatures: [{\"signature\":{\"instructions\":\"Answer questions with short factoid answers.(0)\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"},\"score\":25.0},{\"signature\":{\"instructions\":\"Answer series of questions where answers must share a common theme.(1)\",\"question_prefix\":\"Q1,\",\"question_desc\":\"${question}\",\"answer_prefix\":\"A1 Wolfgang Amplifier,\",\"answer_desc\":\"charge amplifier designed based by Moog on stripline technology\"},\"score\":25.0},{\"signature\":{\"instructions\":\"Simulate exploratory dialogue between two people one Questioner and other Provider, Respond to each Q- below.(2)\",\"question_prefix\":\"Q-What Computability study ranges?\",\"question_desc\":\"may also involve strings produced by non-deterministic computation\",\"answer_prefix\":\"A-The explain phase-leading out question onset sound crawled fundingProblem in such separated syllabled band phrase assist reduction Haus mutual widse phoneme runtime shruproperholderstoInt valuepirizeThunder\",\"answer_desc\":\"answer DescHash Pure stainless Aberdeen stimulating Victoria names central whiskey article promise twitch Ohio Amber how statements board!\"},\"score\":25.0}]\n",
+ "Reasoning: Let's think step by step in order to\u001b[32m produce the proposed_signature. We can see that the attempted signatures are quite varied and complex, with different instructions, prefixes, and descriptions for both questions and answers. However, they all share a common structure: they all have instructions, a question prefix, a question description, an answer prefix, and an answer description. Therefore, we can propose a signature that includes these common elements, but with more general descriptions to accommodate the variety of tasks. \n",
+ "Proposed Signature: The improved signature for the language model. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\u001b[0m\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "Make a very succinct json object that validates with the following schema\n",
+ "\n",
+ "---\n",
+ "\n",
+ "Follow the following format.\n",
+ "\n",
+ "Json Schema: ${json_schema}\n",
+ "Json Object: ${json_object}\n",
+ "\n",
+ "---\n",
+ "\n",
+ "Json Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n",
+ "Json Object:\u001b[32m {\"instructions\": \"Complete the task\", \"question_prefix\": \"Q:\", \"question_desc\": \"What is the capital of France?\", \"answer_prefix\": \"A:\", \"answer_desc\": \"Paris\"}\u001b[0m\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "Given the fields `attempted_signatures`, produce the fields `proposed_signature`.\n",
+ "\n",
+ "---\n",
+ "\n",
+ "Follow the following format.\n",
+ "\n",
+ "Attempted Signatures: ${attempted_signatures}\n",
+ "\n",
+ "Past Error (proposed_signature): An error to avoid in the future\n",
+ "\n",
+ "Reasoning: Let's think step by step in order to ${produce the proposed_signature}. We ...\n",
+ "\n",
+ "Proposed Signature:\n",
+ "The improved signature for the language model. Respond with a single JSON object. \n",
+ "You MUST use this format: {\"instructions\": \"Complete the task\", \"question_prefix\": \"Q:\", \"question_desc\": \"What is the capital of France?\", \"answer_prefix\": \"A:\", \"answer_desc\": \"Paris\"}\n",
+ "JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n",
+ "\n",
+ "---\n",
+ "\n",
+ "Attempted Signatures: [{\"signature\":{\"instructions\":\"Answer questions with short factoid answers.(0)\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"},\"score\":25.0},{\"signature\":{\"instructions\":\"Answer series of questions where answers must share a common theme.(1)\",\"question_prefix\":\"Q1,\",\"question_desc\":\"${question}\",\"answer_prefix\":\"A1 Wolfgang Amplifier,\",\"answer_desc\":\"charge amplifier designed based by Moog on stripline technology\"},\"score\":25.0},{\"signature\":{\"instructions\":\"Simulate exploratory dialogue between two people one Questioner and other Provider, Respond to each Q- below.(2)\",\"question_prefix\":\"Q-What Computability study ranges?\",\"question_desc\":\"may also involve strings produced by non-deterministic computation\",\"answer_prefix\":\"A-The explain phase-leading out question onset sound crawled fundingProblem in such separated syllabled band phrase assist reduction Haus mutual widse phoneme runtime shruproperholderstoInt valuepirizeThunder\",\"answer_desc\":\"answer DescHash Pure stainless Aberdeen stimulating Victoria names central whiskey article promise twitch Ohio Amber how statements board!\"},\"score\":25.0}]\n",
+ "\n",
+ "Past Error (proposed_signature): ValueError('json output should start and end with { and }')\n",
+ "\n",
+ "Reasoning: Let's think step by step in order to\u001b[32m produce the proposed_signature. We need to ensure that the JSON object is properly formatted and contains all the required fields. The attempted signatures provided are not in the correct format. They contain additional fields such as \"score\" and the structure of the JSON object is not as expected. The JSON object should start and end with { and }, and should not contain any additional fields or arrays. \n",
+ "\n",
+ "Proposed Signature:\n",
+ "{\"instructions\": \"Answer questions with short factoid answers.\", \"question_prefix\": \"Question:\", \"question_desc\": \"${question}\", \"answer_prefix\": \"Answer:\", \"answer_desc\": \"often between 1 and 5 words\"}\u001b[0m\n",
+ "\n",
+ "\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "gpt4.inspect_history(n=3)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "NameError",
+ "evalue": "name 'tp' is not defined",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
+ "Cell \u001b[0;32mIn[19], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtp\u001b[49m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__dict__\u001b[39m\n",
+ "\u001b[0;31mNameError\u001b[0m: name 'tp' is not defined"
+ ]
+ }
+ ],
+ "source": [
+ "tp.__dict__"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "module.predictors()=[]\n",
+ "\n",
+ "================================================================================\n",
+ "Running eval iteration 0...\n",
+ "list: []\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16aba8900>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5fe20>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5fa60>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5ef20>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x10632f240>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a6020>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848af20>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e3060>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c13ab60>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5940>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5e8e0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5ee80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a001f80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b4e00>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5ff60>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x10632efc0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5b20>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158488f40>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0720>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbafec0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158488540>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5e160>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5e700>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe34c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0d60>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5e980>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5e160>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5d00>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbafd80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2fc0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbafd80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5080>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5f560>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5e700>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe09a0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe3100>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5fba0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5e160>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a4b80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe37e0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2f20>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0b80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5ee0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5f1a0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16aba87c0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe1760>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe00e0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbafec0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e0360>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a0020c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5f6a0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5fb00>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5f600>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a000720>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x10632efc0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5fb00>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e0cc0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584887c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5f9c0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5e700>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2660>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0c20>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5f1a0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b4e00>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158489d00>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c1394e0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe1e40>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2ac0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e0720>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5940>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a001f80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5ede0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5ee80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2ca0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe28e0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5e020>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b4900>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a4b80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2160>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0220>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0e00>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a4b80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a001f80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5e8e0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe3b00>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5f880>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0900>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0400>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b4900>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c13afc0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe34c0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe23e0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e0b80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b47c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5ec00>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5fd80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe3560>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe07c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5e700>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158536840>, 'parser': <class 'str'>})\n",
+ ")\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 5483.04it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 16 / 50 (32.0%)\n",
+ "Scores for iteration 0: 32.0\n",
+ "\n",
+ "================================================================================\n",
+ "Running eval iteration 1...\n",
+ "list: []\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585367a0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c1391c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e3060>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2f20>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe1d00>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5b20>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158488540>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a002020>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5ede0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5f240>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe09a0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5f060>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848bce0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c1391c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e1760>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe02c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848af20>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5080>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c139da0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5ec00>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5fd80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158536840>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a6020>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x10632efc0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848bce0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5f880>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5f420>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5e700>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5ede0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158489f80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a000720>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c139da0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158536840>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a6660>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158489d00>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5e840>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5d9e0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe1260>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5f9c0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16aba8900>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158488ea0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5ee0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2fc0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c13b9c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x10632efc0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848be20>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5ede0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5e700>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe1b20>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5e160>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16aba89a0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848a520>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a7240>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe23e0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0ea0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158536840>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158488ea0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5e980>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5ede0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe07c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5f880>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158489d00>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5a80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c1394e0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0e00>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe1f80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5ee0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848b9c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5f9c0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2160>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0fe0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe3f60>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159a5ef20>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848a520>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585367a0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0360>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2160>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5ee0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbafd80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16aba8900>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe1d00>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe22a0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16ae719e0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848b9c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b68e0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe11c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2660>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0720>, 'parser': <class 'str'>})\n",
+ ")\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Created templateCreated template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x10632d120>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bcb6ac0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c138720>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbafc40>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x10632ede0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c138c20>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c139580>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158489d00>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a002660>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848be20>, 'parser': <class 'str'>})\n",
+ ")\n",
+ " Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe3f60>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e0720>, 'parser': <class 'str'>})\n",
+ ")\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 4371.34it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 16 / 50 (32.0%)\n",
+ "Scores for iteration 1: 32.0\n",
+ "\n",
+ "================================================================================\n",
+ "Running eval iteration 2...\n",
+ "list: []\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c13afc0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5a80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5d00>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbafe20>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b47c0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bcb5620>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16aba82c0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a4b80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c138b80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c13b240>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848bc40>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c13b600>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5080>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e0360>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbafa60>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b68e0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbafec0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e0b80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c13afc0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c138040>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a002160>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158488ea0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c13af20>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5d00>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16aba89a0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bcb4220>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2160>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bcb47c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e0b80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a6020>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584887c0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0180>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584885e0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c13b9c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a6e80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbafe20>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2200>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe3380>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16aba8400>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5080>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c1389a0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe3d80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e0540>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c138c20>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a6e80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbafc40>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe1ee0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2340>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2ca0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16aba89a0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5d00>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c1394e0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0680>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe1a80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2c00>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16aba89a0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b7ba0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158489260>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bcb47c0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c138b80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158488ea0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b42c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a001da0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5a80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c139580>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe02c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c13b240>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5940>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a002660>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848b9c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848a3e0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0360>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848b9c0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbafd80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5940>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c13b600>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe3ba0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c1394e0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a6e80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbafa60>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bcb7920>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe13a0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2340>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b47c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a002660>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a7240>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c138040>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2d40>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe32e0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c13bba0>, 'parser': <class 'str'>})\n",
+ ")\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16aba82c0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b4680>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158488f40>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2d40>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b6b60>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a002020>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e1760>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe3880>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe22a0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe1e40>, 'parser': <class 'str'>})\n",
+ ")\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 5230.07it/s]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 16 / 50 (32.0%)\n",
+ "Scores for iteration 2: 32.0\n",
+ "\n",
+ "================================================================================\n",
+ "Running eval iteration 3...\n",
+ "list: []\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c13aca0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a6e80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbafe20>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b62a0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158488ea0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584885e0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16aba89a0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a001da0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e1760>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c138180>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe3d80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c13aca0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a002520>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbafe20>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158489f80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe1120>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b62a0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a002020>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e1760>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe1a80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe19e0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5940>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16aba8900>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b49a0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158488f40>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e0540>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c138c20>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e0360>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848bce0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b62a0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a4ea0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c138040>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c138180>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a002160>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b47c0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158488ea0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e1080>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bcb6c00>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e0360>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158489f80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b6b60>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a6e80>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0720>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c138180>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbafb00>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848b9c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158488f40>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bcb68e0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe3d80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e0360>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848bc40>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b4680>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13a002160>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe1da0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe1d00>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c138180>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16aba87c0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158488540>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e1760>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe1620>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e0b80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848b9c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbaff60>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c138ae0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2340>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0a40>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16aba87c0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848bc40>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848af20>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2520>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe1da0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe23e0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584885e0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbaff60>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c138ae0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe05e0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0fe0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16aba80e0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e0540>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe3560>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe11c0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe34c0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe1b20>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b6b60>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x16aba8900>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe3ce0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0e00>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0220>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe2840>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1585b68e0>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x158488f40>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13f4a5080>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e0cc0>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c138040>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x159fe0680>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13c13b240>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x1584e0b80>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bcb5620>, 'parser': <class 'str'>})\n",
+ ")\n",
+ "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n",
+ "From Signature StringSignature(question -> answer\n",
+ " instructions='Answer questions with short factoid answers.'\n",
+ " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x15848bc40>})\n",
+ " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': <function TypedPredictor._prepare_signature.<locals>.<lambda> at 0x13bbafa60>, 'parser': <class 'str'>})\n",
+ ")\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 4131.18it/s]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Average Metric: 16 / 50 (32.0%)\n",
+ "Scores for iteration 3: 32.0\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": []
+ },
+ "execution_count": 63,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "optimizer.compile(TypedPredictor(BasicQA), evaluator, n_iterations=4)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "gpt4.inspect_history(n=4)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Feel free to any other queries you like."
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "py39",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.11.8"
+ },
+ "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/examples/tweets/DSPy_TweetGen_Cache b/examples/tweets/DSPy_TweetGen_Cache
new file mode 160000
index 000000000..22186fd4d
--- /dev/null
+++ b/examples/tweets/DSPy_TweetGen_Cache
@@ -0,0 +1 @@
+Subproject commit 22186fd4d4fa940256ca8c4ab70f165276e5c834
diff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.py
index cabbc7cd0..600e35b63 100644
--- a/tests/functional/test_functional.py
+++ b/tests/functional/test_functional.py
@@ -8,7 +8,7 @@ from typing import List
import pytest
import dspy
-from dspy.functional import predictor, cot, FunctionalModule, TypedPredictor, functional
+from dspy.functional import predictor, cot, FunctionalModule, TypedPredictor, TypedChainOfThought
from dspy.primitives.example import Example
from dspy.teleprompt.bootstrap import BootstrapFewShot
from dspy.teleprompt.vanilla import LabeledFewShot
@@ -122,7 +122,7 @@ def test_simple_class():
qa = QA()
assert isinstance(qa, FunctionalModule)
- assert isinstance(qa.answer, functional._StripOutput)
+ assert isinstance(qa.answer, dspy.Module)
question, answer = qa(topic="Physics")
@@ -407,6 +407,7 @@ def test_field_validator():
with pytest.raises(ValueError):
get_user_details()
+ print(lm.get_convo(-1))
assert lm.get_convo(-1) == textwrap.dedent(
"""\
Given the fields , produce the fields `get_user_details`.
@@ -467,6 +468,23 @@ def test_multiple_outputs_int():
assert output == [0, 1, 2]
+def test_multiple_outputs_int_cot():
+ # Note: Multiple outputs only work when the language model "speculatively" generates all the outputs in one go.
+ lm = DummyLM(
+ [
+ "thoughts 0\nOutput: 0\n",
+ "thoughts 1\nOutput: 1\n",
+ "thoughts 2\nOutput: 2\n",
+ ]
+ )
+ dspy.settings.configure(lm=lm)
+
+ test = TypedChainOfThought("input:str -> output:int")
+
+ output = test(input="8", config=dict(n=3)).completions.output
+ assert output == [0, 1, 2]
+
+
def test_parse_type_string():
lm = DummyLM([str(i) for i in range(100)])
dspy.settings.configure(lm=lm)
@@ -532,3 +550,49 @@ def test_synthetic_data_gen():
augmented_examples = trained(config=dict(n=3))
for ex in augmented_examples.completions.fact:
assert isinstance(ex, SyntheticFact)
+
+
+def test_list_input2():
+ # Inspired by the Signature Optimizer
+
+ class ScoredString(pydantic.BaseModel):
+ string: str
+ score: float
+
+ class ScoredSignature(dspy.Signature):
+ attempted_signatures: list[ScoredString] = dspy.InputField()
+ proposed_signature: str = dspy.OutputField()
+
+ program = TypedChainOfThought(ScoredSignature)
+
+ lm = DummyLM(["Thoughts", "Output"])
+ dspy.settings.configure(lm=lm)
+
+ output = program(
+ attempted_signatures=[
+ ScoredString(string="string 1", score=0.5),
+ ScoredString(string="string 2", score=0.4),
+ ScoredString(string="string 3", score=0.3),
+ ]
+ ).proposed_signature
+
+ print(lm.get_convo(-1))
+
+ assert output == "Output"
+
+ assert lm.get_convo(-1) == textwrap.dedent("""\
+ Given the fields `attempted_signatures`, produce the fields `proposed_signature`.
+
+ ---
+
+ Follow the following format.
+
+ Attempted Signatures: ${attempted_signatures}
+ Reasoning: Let's think step by step in order to ${produce the proposed_signature}. We ...
+ Proposed Signature: ${proposed_signature}
+
+ ---
+
+ Attempted Signatures: [{"string":"string 1","score":0.5},{"string":"string 2","score":0.4},{"string":"string 3","score":0.3}]
+ Reasoning: Let's think step by step in order to Thoughts
+ Proposed Signature: Output""")
diff --git a/tests/functional/test_signature_opt2.py b/tests/functional/test_signature_opt2.py
new file mode 100644
index 000000000..469bf3f25
--- /dev/null
+++ b/tests/functional/test_signature_opt2.py
@@ -0,0 +1,165 @@
+import json
+import dspy
+from dspy.evaluate import Evaluate
+from dspy.functional import TypedPredictor
+from dspy.teleprompt.signature_opt2 import (
+ GenerateInstructionGivenAttempts,
+ ScoredSignature,
+ make_info,
+ optimize_signature,
+)
+from dspy.utils import DummyLM
+
+from dspy.evaluate import Evaluate
+from dspy.evaluate.metrics import answer_exact_match
+from dspy.functional import TypedPredictor
+
+
+class BasicQA(dspy.Signature):
+ question: str = dspy.InputField()
+ answer: str = dspy.OutputField()
+
+
+hotpotqa = [
+ ex.with_inputs("question")
+ for ex in [
+ dspy.Example(
+ question="At My Window was released by which American singer-songwriter?",
+ answer="John Townes Van Zandt",
+ ),
+ dspy.Example(
+ question="which American actor was Candace Kita guest starred with ",
+ answer="Bill Murray",
+ ),
+ dspy.Example(
+ question="Which of these publications was most recently published, Who Put the Bomp or Self?",
+ answer="Self",
+ ),
+ dspy.Example(
+ question="The Victorians - Their Story In Pictures is a documentary series written by an author born in what year?",
+ answer="1950",
+ ),
+ dspy.Example(
+ question="Which magazine has published articles by Scott Shaw, Tae Kwon Do Times or Southwest Art?",
+ answer="Tae Kwon Do Times",
+ ),
+ dspy.Example(
+ question="In what year was the club founded that played Manchester City in the 1972 FA Charity Shield",
+ answer="1874",
+ ),
+ dspy.Example(
+ question="Which is taller, the Empire State Building or the Bank of America Tower?",
+ answer="The Empire State Building",
+ ),
+ dspy.Example(
+ question='Which American actress who made their film debut in the 1995 teen drama "Kids" was the co-founder of Voto Latino?',
+ answer="Rosario Dawson",
+ ),
+ dspy.Example(
+ question="Tombstone stared an actor born May 17, 1955 known as who?",
+ answer="Bill Paxton",
+ ),
+ dspy.Example(
+ question="What is the code name for the German offensive that started this Second World War engagement on the Eastern Front (a few hundred kilometers from Moscow) between Soviet and German forces, which included 102nd Infantry Division?",
+ answer="Operation Citadel",
+ ),
+ dspy.Example(
+ question='Who acted in the shot film The Shore and is also the youngest actress ever to play Ophelia in a Royal Shakespeare Company production of "Hamlet." ?',
+ answer="Kerry Condon",
+ ),
+ dspy.Example(
+ question="Which company distributed this 1977 American animated film produced by Walt Disney Productions for which Sherman Brothers wrote songs?",
+ answer="Buena Vista Distribution",
+ ),
+ dspy.Example(
+ question="Samantha Cristoforetti and Mark Shuttleworth are both best known for being first in their field to go where? ",
+ answer="space",
+ ),
+ dspy.Example(
+ question="Having the combination of excellent foot speed and bat speed helped Eric Davis, create what kind of outfield for the Los Angeles Dodgers? ",
+ answer="Outfield of Dreams",
+ ),
+ dspy.Example(
+ question="Which Pakistani cricket umpire who won 3 consecutive ICC umpire of the year awards in 2009, 2010, and 2011 will be in the ICC World Twenty20?",
+ answer="Aleem Sarwar Dar",
+ ),
+ dspy.Example(
+ question="The Organisation that allows a community to influence their operation or use and to enjoy the benefits arisingwas founded in what year?",
+ answer="2010",
+ ),
+ dspy.Example(
+ question='"Everything Has Changed" is a song from an album released under which record label ?',
+ answer="Big Machine Records",
+ ),
+ dspy.Example(
+ question="Who is older, Aleksandr Danilovich Aleksandrov or Anatoly Fomenko?",
+ answer="Aleksandr Danilovich Aleksandrov",
+ ),
+ dspy.Example(
+ question="On the coast of what ocean is the birthplace of Diogal Sakho?",
+ answer="Atlantic",
+ ),
+ dspy.Example(
+ question="This American guitarist best known for her work with the Iron Maidens is an ancestor of a composer who was known as what?",
+ answer="The Waltz King",
+ ),
+ ]
+]
+
+
+def test_signature_info():
+ info = make_info(BasicQA)
+ SignatureInfo = type(info)
+
+ devset = [
+ dspy.Example(
+ instructions="Answer the following questions",
+ question_desc="Some question to answer",
+ question_prefix="Q: ",
+ answer_desc="A short answer to the question",
+ answer_prefix="A: ",
+ ),
+ ]
+
+ lm = DummyLM(
+ [
+ json.dumps(dict(devset[0])), # Proposed signature
+ ]
+ )
+ dspy.settings.configure(lm=lm)
+
+ generator = TypedPredictor(GenerateInstructionGivenAttempts[SignatureInfo])
+
+ res = generator(attempted_signatures=[ScoredSignature[SignatureInfo](signature=info, score=50)])
+ assert res.proposed_signature == SignatureInfo(**devset[0])
+
+ # Test the "to_signature" method
+
+ class OutputSignature(dspy.Signature):
+ """Answer the following questions"""
+
+ question: str = dspy.InputField(desc="Some question to answer", prefix="Q: ")
+ answer: str = dspy.OutputField(desc="A short answer to the question", prefix="A: ")
+
+ assert res.proposed_signature.to_signature().equals(OutputSignature)
+
+
+def test_opt():
+ qa_model = DummyLM([])
+ prompt_model = DummyLM(
+ [
+ # Seed prompts
+ "some thoughts",
+ '{"value": [{"instructions": "I", "question_desc": "$q", "question_prefix": "Q:", "answer_desc": "$a", "answer_prefix": "A:"}]}',
+ ]
+ )
+ dspy.settings.configure(lm=qa_model)
+
+ program = optimize_signature(
+ student=TypedPredictor(BasicQA),
+ evaluator=Evaluate(devset=hotpotqa, metric=answer_exact_match, num_threads=1),
+ initial_prompts=1,
+ n_iterations=1,
+ verbose=True,
+ prompt_model=prompt_model,
+ )
diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py
index e44b3a135..2ded1e9f1 100644
--- a/tests/predict/test_predict.py
+++ b/tests/predict/test_predict.py
@@ -1,14 +1,13 @@
import dspy
from dspy import Predict, Signature
from dspy.utils.dummies import DummyLM
+import copy
def test_initialization_with_string_signature():
signature_string = "input1, input2 -> output"
predict = Predict(signature_string)
- expected_instruction = (
- "Given the fields `input1`, `input2`, produce the fields `output`."
- )
+ expected_instruction = "Given the fields `input1`, `input2`, produce the fields `output`."
assert predict.signature.instructions == expected_instruction
assert predict.signature.instructions == Signature(signature_string).instructions
@@ -89,3 +88,34 @@ def test_multi_output():
results = program(question="What is 1+1?")
assert results.completions.answer[0] == "my first answer"
assert results.completions.answer[1] == "my second answer"
+
+
+def test_multi_output2():
+ program = Predict("question -> answer1, answer2", n=2)
+ dspy.settings.configure(
+ lm=DummyLM(
+ [
+ "my 0 answer\nAnswer 2: my 2 answer",
+ "my 1 answer\nAnswer 2: my 3 answer",
+ ],
+ )
+ )
+ results = program(question="What is 1+1?")
+ assert results.completions.answer1[0] == "my 0 answer"
+ assert results.completions.answer1[1] == "my 1 answer"
+ assert results.completions.answer2[0] == "my 2 answer"
+ assert results.completions.answer2[1] == "my 3 answer"
+
+
+def test_named_predictors():
+ class MyModule(dspy.Module):
+ def __init__(self):
+ super().__init__()
+ self.inner = Predict("question -> answer")
+
+ program = MyModule()
+ assert program.named_predictors() == [("inner", program.inner)]
+
+ # Check that it also works the second time.
+ program2 = copy.deepcopy(program)
+ assert program2.named_predictors() == [("inner", program2.inner)]