Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Initial version of ExperimentalAdapter

Omar Khattabfcfc0ad263a5
diff --git a/dsp/__init__.py b/dsp/__init__.py
index 4f98315f1..f97fa2c7e 100644
--- a/dsp/__init__.py
+++ b/dsp/__init__.py
@@ -1,6 +1,6 @@
from .modules import *
from .primitives import *
-from .templates import *
+from .adapters import *
from .utils import settings
"""
diff --git a/dsp/adapters/__init__.py b/dsp/adapters/__init__.py
new file mode 100644
index 000000000..9afab2787
--- /dev/null
+++ b/dsp/adapters/__init__.py
@@ -0,0 +1,5 @@
+from .base_template import *
+from .template import *
+from .experimental_adapter import *
+from .utils import *
+
diff --git a/dsp/templates/template_v3.py b/dsp/adapters/base_template.py
similarity index 91%
rename from dsp/templates/template_v3.py
rename to dsp/adapters/base_template.py
index 605705c15..4c3733a23 100644
--- a/dsp/templates/template_v3.py
+++ b/dsp/adapters/base_template.py
@@ -1,6 +1,10 @@
from typing import Callable
+from collections import namedtuple
-from dsp.templates import Field, TemplateV2, format_answers, passages2text
+from .utils import format_answers, passages2text
+
+
+Field = namedtuple("Field", "name separator input_variable output_variable description")
class Type:
@@ -19,7 +23,7 @@ class Type:
return isinstance(__value, Type) and self.__dict__ == __value.__dict__
-class Template(TemplateV2):
+class BaseTemplate:
"""A template datatype that represents the structure of communicate with the LM."""
def __init__(self, instructions: str, **kwargs):
@@ -61,9 +65,7 @@ class Template(TemplateV2):
v1, v2 = self.kwargs[k], other.kwargs[k]
if not v1 == v2:
print(k, v1, v2)
-
- # print("here?", self.instructions == other.instructions, self.kwargs == other.kwargs)
return self.instructions == other.instructions and self.kwargs == other.kwargs
def __str__(self) -> str:
diff --git a/dsp/adapters/experimental_adapter.py b/dsp/adapters/experimental_adapter.py
new file mode 100644
index 000000000..f923a5e36
--- /dev/null
+++ b/dsp/adapters/experimental_adapter.py
@@ -0,0 +1,253 @@
+from typing import Any, Union
+
+import dsp
+from dsp.primitives.demonstrate import Example
+from .base_template import BaseTemplate
+
+
+
+class ExperimentalAdapter(BaseTemplate):
+ def query(self, example: Example, is_demo: bool = False) -> str:
+ """Retrieves the input variables from the example and formats them into a query string."""
+ result: list[str] = []
+
+ # If not a demo, find the last field that doesn't have a value set in `example` and set it to ""
+ # This creates the "Output:" prefix at the end of the prompt.
+ if not is_demo:
+ has_value = [
+ field.input_variable in example
+ and example[field.input_variable] is not None
+ and example[field.input_variable] != ""
+ for field in self.fields
+ ]
+
+ # If there are no inputs, set the first field to ""
+ if not any(has_value):
+ example[self.fields[0].input_variable] = ""
+ # Otherwise find the first field without a value.
+ else:
+ for i in range(1, len(has_value)):
+ if has_value[i - 1] and not any(has_value[i:]):
+ example[self.fields[i].input_variable] = ""
+ break
+
+ for field in self.fields:
+ if field.input_variable in example and example[field.input_variable] is not None:
+ if field.input_variable in self.format_handlers:
+ format_handler = self.format_handlers[field.input_variable]
+ else:
+
+ def format_handler(x):
+ assert type(x) == str, f"Need format_handler for {field.input_variable} of type {type(x)}"
+ return " ".join(x.split())
+
+ formatted_value = format_handler(example[field.input_variable])
+ separator = "\n" if field.separator == " " and "\n" in formatted_value else field.separator
+
+ result.append(
+ f"{field.name}{separator}{formatted_value}",
+ )
+
+ if self._has_augmented_guidelines() and (example.get("augmented", False)):
+ return "\n\n".join([r for r in result if r])
+ return "\n".join([r for r in result if r])
+
+ def guidelines(self, show_guidelines=True) -> str:
+ """Returns the task guidelines as described in the lm prompt"""
+ if (not show_guidelines) or (hasattr(dsp.settings, "show_guidelines") and not dsp.settings.show_guidelines):
+ return ""
+
+ result = "Follow the following format.\n\n"
+
+ example = dsp.Example()
+ for field in self.fields:
+ example[field.input_variable] = field.description
+ example.augmented = self._has_augmented_guidelines()
+
+ result += self.query(example)
+ return result
+
+ def _has_augmented_guidelines(self):
+ return len(self.fields) > 3 or any(
+ ("\n" in field.separator) or ("\n" in field.description) for field in self.fields
+ )
+
+ def extract(
+ self,
+ example: Union[Example, dict[str, Any]],
+ raw_pred: str,
+ ) -> Example:
+ """Extracts the answer from the LM raw prediction using the template structure
+
+ Args:
+ example (Union[Example, dict[str, Any]]): Contains the input variables that raw_pred was completed on.
+ raw_pred (str): LM generated string
+
+ Returns:
+ Example: The example with the output variables filled in
+ """
+ example = dsp.Example(example)
+
+ raw_pred = raw_pred.strip()
+ parts = raw_pred.split('\n')
+ adjusted_parts = []
+ for part in parts:
+ trimmed_part = part.strip()
+ if trimmed_part:
+ if adjusted_parts:
+ adjusted_parts.append('\n' + trimmed_part)
+ else:
+ adjusted_parts.append(trimmed_part)
+ raw_pred = '\n'.join(adjusted_parts)
+
+ idx = 0
+ while idx < len(self.fields):
+ if self.fields[idx].input_variable not in example or example[self.fields[idx].input_variable] is None:
+ break
+ idx += 1
+
+ import dspy
+
+ idx = min(idx, len(self.fields) - 1)
+ while raw_pred != "" and idx < len(self.fields):
+ if idx < len(self.fields) - 1:
+ next_field_name = "\n" + self.fields[idx + 1].name
+ offset = raw_pred.find(next_field_name)
+
+ if offset >= 0:
+ if dspy.settings.release >= 20231003:
+ example[self.fields[idx].output_variable] = raw_pred[:offset].strip().rstrip("---").strip()
+ raw_pred = raw_pred[offset + len(next_field_name) :].strip().rstrip("---").strip()
+ else:
+ field_name_parts = self.fields[idx].name.split()
+ start_pos = 0
+ for part in field_name_parts:
+ pos = raw_pred.find(part.strip())
+ if pos != -1:
+ start_pos = pos + len(part)
+ else:
+ break
+
+ example[self.fields[idx].output_variable] = raw_pred[start_pos:offset].strip().rstrip("---").strip()
+ raw_pred = raw_pred[offset + len(next_field_name) :].strip()
+ idx += 1
+ else:
+ if dspy.settings.release >= 20231003:
+ example[self.fields[idx].output_variable] = raw_pred.strip().rstrip("---").strip()
+ else:
+ example[self.fields[idx].output_variable] = raw_pred.strip()
+
+ raw_pred = ""
+ idx += 1
+ break
+
+ else:
+ assert idx == len(self.fields) - 1, (idx, len(self.fields))
+
+ if dspy.settings.release >= 20231003:
+ example[self.fields[idx].output_variable] = raw_pred.strip().rstrip("---").strip()
+ else:
+ field_name_parts = self.fields[idx].name.split()
+ start_pos = 0
+ for part in field_name_parts:
+ pos = raw_pred.find(part.strip())
+ if pos != -1:
+ start_pos = pos + len(part)
+ else:
+ break
+ example[self.fields[idx].output_variable] = raw_pred[start_pos:].strip()
+
+ break
+
+ return example
+
+ def __call__(self, example, show_guidelines=True) -> str:
+ example = dsp.Example(example)
+ output_fields = []
+ for i in range(len(self.fields)):
+ if self.fields[i].input_variable not in example:
+ output_field = self.fields[i].input_variable
+ if output_field not in output_fields:
+ output_fields.append(self.fields[i].name.split(':')[0])
+
+ if hasattr(dsp.settings, "query_only") and dsp.settings.query_only:
+ return self.query(example)
+
+ # The training data should not contain the output variable
+ if self.fields[-1].input_variable in example:
+ del example[self.fields[-1].input_variable]
+
+ rdemos = [
+ self.query(demo, is_demo=True)
+ for demo in example.demos
+ if (
+ (not demo.get("augmented", False))
+ and ( # validate that the training example has the same primitive input var as the template
+ self.fields[-1].input_variable in demo and demo[self.fields[-1].input_variable] is not None
+ )
+ )
+ ]
+
+ ademos = [self.query(demo, is_demo=True) for demo in example.demos if demo.get("augmented", False)]
+
+ # Move the rdemos to ademos if rdemo has all the fields filled in
+ rdemos_ = []
+ new_ademos = []
+ for rdemo in rdemos:
+ if all((field.name in rdemo) for field in self.fields if field.input_variable in example):
+ import dspy
+
+ if dspy.settings.release >= 20230928:
+ new_ademos.append(rdemo)
+ else:
+ ademos.append(rdemo)
+ else:
+ rdemos_.append(rdemo)
+
+ ademos = new_ademos + ademos
+ rdemos = rdemos_
+
+ long_query = self._has_augmented_guidelines()
+
+ if long_query:
+ example["augmented"] = True
+
+ query = self.query(example)
+
+ # if it has more lines than fields
+ if len(query.split("\n")) > len(self.fields):
+ long_query = True
+
+ if not example.get("augmented", False):
+ example["augmented"] = True
+ query = self.query(example)
+
+ rdemos = "\n\n".join(rdemos)
+
+ if len(rdemos) >= 1 and len(ademos) == 0 and not long_query:
+ rdemos_and_query = "\n\n".join([rdemos, query])
+ parts = [
+ self.instructions,
+ self.guidelines(show_guidelines),
+ rdemos_and_query,
+ ]
+ elif len(rdemos) == 0:
+ parts = [
+ self.instructions,
+ self.guidelines(show_guidelines),
+ *ademos,
+ query,
+ ]
+ else:
+ parts = [
+ self.instructions,
+ rdemos,
+ self.guidelines(show_guidelines),
+ *ademos,
+ query,
+ ]
+
+ prompt = "\n\n---\n\n".join([p.strip() for p in parts if p])
+ prompt_ = prompt[: prompt.rfind("\n")].strip()
+ prompt_ += f"\n\nPlease provide the output fields {', '.join(output_fields[:-1]) + (', then ' if len(output_fields) > 2 else ' then ') + output_fields[-1] if len(output_fields) > 1 else output_fields[0]}. Do so immediately, without additional content before or after, and precisely as the format above shows. Always separate the output fields by only 2 newlines. Begin with the field {output_fields[0]}."
+ return prompt_.strip()
diff --git a/dsp/templates/template_v2.py b/dsp/adapters/template.py
similarity index 79%
rename from dsp/templates/template_v2.py
rename to dsp/adapters/template.py
index 0df17185b..a9009ee02 100644
--- a/dsp/templates/template_v2.py
+++ b/dsp/adapters/template.py
@@ -1,73 +1,12 @@
-import re
-from collections import namedtuple
from typing import Any, Union
import dsp
from dsp.primitives.demonstrate import Example
+from .base_template import BaseTemplate
-from .utils import format_answers, passages2text
-Field = namedtuple("Field", "name separator input_variable output_variable description")
-
-# TODO: de-duplicate with dsp/templates/template.py
-
-
-class TemplateV2:
- def __init__(
- self,
- template,
- format_handlers={
- "passages": passages2text,
- "context": passages2text,
- "answer": format_answers,
- "answers": format_answers,
- },
- ):
- self.format_handlers = format_handlers
-
- template = template.strip()
-
- self.instructions = re.search("(.*)\n", template).group(1)
- template = template[len(self.instructions) :].strip()
-
- self.fields = []
- while len(template) > 0:
- match = re.search("(.*)(\s){(.*)}\s(.*\${.*})", template)
- if match is not None:
- name = match.group(1)
- separator = match.group(2)
- variable = match.group(3)
- description = match.group(4)
- else:
- match = re.search("(.*)(\s){(.*)}", template)
- if match is not None:
- name = match.group(1)
- separator = match.group(2)
- variable = match.group(3)
- description = None
- else:
- raise ValueError("Could not parse template")
-
- var_match = re.match("(.*) -> (.*)", variable)
- if var_match is not None:
- input_variable = var_match.group(1)
- output_variable = var_match.group(2)
- else:
- input_variable = variable
- output_variable = variable
-
- self.fields.append(
- Field(
- name=name,
- separator=separator,
- input_variable=input_variable,
- output_variable=output_variable,
- description=description,
- ),
- )
-
- template = template[len(match.group(0)) :].strip()
+class Template(BaseTemplate):
def query(self, example: Example, is_demo: bool = False) -> str:
"""Retrieves the input variables from the example and formats them into a query string."""
result: list[str] = []
diff --git a/dsp/templates/utils.py b/dsp/adapters/utils.py
similarity index 100%
rename from dsp/templates/utils.py
rename to dsp/adapters/utils.py
diff --git a/dsp/primitives/predict.py b/dsp/primitives/predict.py
index 41edf6c51..dcb1b3152 100644
--- a/dsp/primitives/predict.py
+++ b/dsp/primitives/predict.py
@@ -3,7 +3,7 @@ from typing import Any, Callable, Optional
import dsp
from dsp.primitives.demonstrate import Example
-from dsp.templates.template_v3 import Template
+from dsp.adapters import Template
from dsp.utils import normalize_text, zipstar
from dsp.utils.utils import dotdict
@@ -61,9 +61,7 @@ def _generate(template: Template, **kwargs) -> Callable:
generator = dsp.settings.lm
- def do_generate(
- example: Example, stage: str, max_depth: int = 2, original_example=None,
- ):
+ def do_generate(example: Example, stage: str, max_depth: int = 2, original_example=None):
if not dsp.settings.lm:
raise AssertionError("No LM is loaded.")
original_example = original_example or example
@@ -82,9 +80,7 @@ def _generate(template: Template, **kwargs) -> Callable:
last_field_idx = 0
for field_idx, key in enumerate(field_names):
- completions_ = [
- c for c in completions if key in c.keys() and c[key] is not None
- ]
+ completions_ = [c for c in completions if key in c.keys() and c[key] is not None]
# Filter out completions that are missing fields that are present in at least one completion.
if len(completions_):
@@ -100,137 +96,129 @@ def _generate(template: Template, **kwargs) -> Callable:
# Recurse with greedy decoding and a shorter length.
max_tokens = kwargs.get("max_tokens", dsp.settings.lm.kwargs["max_tokens"])
max_tokens = min(max(75, max_tokens // 2), max_tokens)
- new_kwargs = {
- **kwargs,
- "max_tokens": max_tokens,
- "n": 1,
- "temperature": 0.0,
- }
+ new_kwargs = {**kwargs, "max_tokens": max_tokens, "n": 1, "temperature": 0.0,}
assert max_depth > 0
- return generate(template, **new_kwargs)(
- completion,
- stage=stage,
- max_depth=max_depth - 1,
- original_example=original_example,
- )
+ return generate(template, **new_kwargs)(completion, stage=stage,
+ max_depth=max_depth - 1,
+ original_example=original_example,)
completions = Completions(completions, template=template)
example = example.copy(completions=completions)
- if len(completions) == 1:
- completion = completions[0]
- example[stage] = example.copy(**completion)
-
- if dsp.settings.compiling:
- inputs_ = set(original_example.keys())
- inputs = [
- f.input_variable
- for f in template.fields
- if f.input_variable in inputs_
- ]
- outputs = [
- f.output_variable
- for f in template.fields
- if f.input_variable not in inputs_
- ]
-
- example.compiling_stages = example.get("compiling_stages", [])
- example.compiling_stages.append(
- {
- "name": stage,
- "template": template,
- "inputs": inputs,
- "outputs": outputs,
- },
- )
- else:
- # assert not dsp.settings.compiling, "TODO: At this point, cannot compile n>1 generations"
- example[stage] = dotdict(completions=completions)
+ # if len(completions) == 1:
+ # completion = completions[0]
+ # example[stage] = example.copy(**completion)
+
+ # if dsp.settings.compiling:
+ # inputs_ = set(original_example.keys())
+ # inputs = [
+ # f.input_variable
+ # for f in template.fields
+ # if f.input_variable in inputs_
+ # ]
+ # outputs = [
+ # f.output_variable
+ # for f in template.fields
+ # if f.input_variable not in inputs_
+ # ]
+
+ # example.compiling_stages = example.get("compiling_stages", [])
+ # example.compiling_stages.append(
+ # {
+ # "name": stage,
+ # "template": template,
+ # "inputs": inputs,
+ # "outputs": outputs,
+ # },
+ # )
+ # else:
+ # # assert not dsp.settings.compiling, "TODO: At this point, cannot compile n>1 generations"
+ # example[stage] = dotdict(completions=completions)
return example, completions
return do_generate
-def generate_sc(
- example, prompt, normalize=True, extract=None, prediction_field=None, **kwargs,
-):
- if not dsp.settings.lm:
- raise AssertionError("No LM is loaded.")
- kwargs = {"temperature": 0.7, "n": 20, "max_tokens": 150, **kwargs}
+# def generate_sc(
+# example, prompt, normalize=True, extract=None, prediction_field=None, **kwargs,
+# ):
+# if not dsp.settings.lm:
+# raise AssertionError("No LM is loaded.")
+# kwargs = {"temperature": 0.7, "n": 20, "max_tokens": 150, **kwargs}
- completions = dsp.settings.lm(prompt, **kwargs)
- completions = extract_final_answer(example, completions, extract=extract)
- return majority_vote_(
- completions, normalize=normalize, prediction_field=prediction_field,
- )
+# completions = dsp.settings.lm(prompt, **kwargs)
+# completions = extract_final_answer(example, completions, extract=extract)
+# return majority_vote_(
+# completions, normalize=normalize, prediction_field=prediction_field,
+# )
-def extract_final_answer(example, completions, extract=None):
- if not dsp.settings.lm:
- raise AssertionError("No LM is loaded.")
- if extract:
- completions = [extract(example, p) for p in completions]
- else:
- completions = [
- p.strip().split("\n")[-1].split(":", 1)[-1].strip() for p in completions
- ]
+# def extract_final_answer(example, completions, extract=None):
+# if not dsp.settings.lm:
+# raise AssertionError("No LM is loaded.")
+# if extract:
+# completions = [extract(example, p) for p in completions]
+# else:
+# completions = [
+# p.strip().split("\n")[-1].split(":", 1)[-1].strip() for p in completions
+# ]
- # TODO: make thread-safe?
- dsp.settings.lm.history.append(
- {**dsp.settings.lm.history[-1], "completions": completions},
- )
+# # TODO: make thread-safe?
+# dsp.settings.lm.history.append(
+# {**dsp.settings.lm.history[-1], "completions": completions},
+# )
- return completions
+# return completions
-def majority(
- completions: Completions, normalize: bool = True, field: Optional[str] = None,
-):
- """Returns the most common completion for the target field or the last field in the template."""
- field = completions.template.fields[-1].output_variable if field is None else field
+# def majority(
+# completions: Completions, normalize: bool = True, field: Optional[str] = None,
+# ):
+# """Returns the most common completion for the target field or the last field in the template."""
+# field = completions.template.fields[-1].output_variable if field is None else field
- return Completions(
- majority_vote_(completions, normalize=normalize, prediction_field=field),
- template=completions.template,
- )
+# return Completions(
+# majority_vote_(completions, normalize=normalize, prediction_field=field),
+# template=completions.template,
+# )
-def majority_vote_(completions: Completions, normalize: bool, prediction_field: str):
- """Core logic for majority vote."""
+# def majority_vote_(completions: Completions, normalize: bool, prediction_field: str):
+# """Core logic for majority vote."""
- if not dsp.settings.lm:
- raise AssertionError("No LM is loaded.")
+# if not dsp.settings.lm:
+# raise AssertionError("No LM is loaded.")
- normalized_to_original = {}
- if normalize:
- original_completions = completions
- completions_ = []
- for pred in completions:
- if prediction_field in pred:
- completions_.append(normalize_text(pred[prediction_field]))
- else:
- completions_.append("")
- completions = completions_
+# normalized_to_original = {}
+# if normalize:
+# original_completions = completions
+# completions_ = []
+# for pred in completions:
+# if prediction_field in pred:
+# completions_.append(normalize_text(pred[prediction_field]))
+# else:
+# completions_.append("")
+# completions = completions_
- for completion, normalized_completion in zip(original_completions, completions):
- if normalized_completion not in normalized_to_original:
- normalized_to_original[normalized_completion] = completion
+# for completion, normalized_completion in zip(original_completions, completions):
+# if normalized_completion not in normalized_to_original:
+# normalized_to_original[normalized_completion] = completion
- completions_ = [x for x in completions if x]
+# completions_ = [x for x in completions if x]
- if completions_:
- completions = completions_
+# if completions_:
+# completions = completions_
- topk = Counter(completions).most_common()
- pred, _ = topk[0]
+# topk = Counter(completions).most_common()
+# pred, _ = topk[0]
- if normalize:
- pred = normalized_to_original[pred]
+# if normalize:
+# pred = normalized_to_original[pred]
- dsp.settings.lm.history.append(
- {**dsp.settings.lm.history[-1], "topk": topk, "completions": [pred]},
- )
+# dsp.settings.lm.history.append(
+# {**dsp.settings.lm.history[-1], "topk": topk, "completions": [pred]},
+# )
- return [pred]
+# return [pred]
diff --git a/dsp/templates/__init__.py b/dsp/templates/__init__.py
deleted file mode 100644
index 5eaae95f7..000000000
--- a/dsp/templates/__init__.py
+++ /dev/null
@@ -1,4 +0,0 @@
-from .template_v2 import *
-from .template_v3 import *
-from .utils import *
-
diff --git a/dsp/utils/settings.py b/dsp/utils/settings.py
index 716375b70..4396d5d16 100644
--- a/dsp/utils/settings.py
+++ b/dsp/utils/settings.py
@@ -32,7 +32,7 @@ class Settings:
reranker=None,
compiled_lm=None,
force_reuse_cached_compilation=False,
- compiling=False,
+ compiling=False, # TODO: can probably be removed
skip_logprobs=False,
trace=[],
release=0,
@@ -41,6 +41,7 @@ class Settings:
assert_failures=0,
suggest_failures=0,
langchain_history=[],
+ experimental=False
)
cls._instance.__append(config)
diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py
index 8b0025b7a..200354769 100644
--- a/dspy/functional/functional.py
+++ b/dspy/functional/functional.py
@@ -8,7 +8,7 @@ import ujson
from pydantic.fields import FieldInfo
import dspy
-from dsp.templates import passages2text
+from dsp.adapters import passages2text
from dspy.primitives.prediction import Prediction
from dspy.signatures.signature import ensure_signature, make_signature
diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py
index 332560094..8ce7741a9 100644
--- a/dspy/predict/predict.py
+++ b/dspy/predict/predict.py
@@ -61,6 +61,8 @@ class Predict(Parameter):
return self.forward(**kwargs)
def forward(self, **kwargs):
+ assert not dsp.settings.compiling, "It's no longer ever the case that .compiling is True"
+
# Extract the three privileged keyword arguments.
new_signature = ensure_signature(kwargs.pop("new_signature", None))
signature = ensure_signature(kwargs.pop("signature", self.signature))
@@ -83,11 +85,6 @@ class Predict(Parameter):
config["temperature"] = 0.7
# print(f"#> Setting temperature to 0.7 since n={num_generations} and prior temperature={temperature}.")
- # All of the other kwargs are presumed to fit a prefix of the signature.
- # That is, they are input variables for the bottom most generation, so
- # we place them inside the input - x - together with the demos.
- x = dsp.Example(demos=demos, **kwargs)
-
if new_signature is not None:
signature = new_signature
@@ -96,29 +93,10 @@ class Predict(Parameter):
missing = [k for k in signature.input_fields if k not in kwargs]
print(f"WARNING: Not all input fields were provided to module. Present: {present}. Missing: {missing}.")
- # Switch to legacy format for dsp.generate
- template = signature_to_template(signature)
-
- if self.lm is None:
- x, C = dsp.generate(template, **config)(x, stage=self.stage)
+ if dsp.settings.experimental:
+ completions = new_generate(lm, signature, dsp.Example(demos=demos, **kwargs), **config)
else:
- # Note: query_only=True means the instructions and examples are not included.
- # I'm not really sure why we'd want to do that, but it's there.
- with dsp.settings.context(lm=self.lm, query_only=True):
- x, C = dsp.generate(template, **config)(x, stage=self.stage)
-
- assert self.stage in x, "The generated (input, output) example was not stored"
-
- completions = []
-
- for c in C:
- completions.append({})
- for field in template.fields:
- if field.output_variable not in kwargs.keys():
- completions[-1][field.output_variable] = getattr(
- c,
- field.output_variable,
- )
+ completions = old_generate(demos, signature, kwargs, config, self.lm, self.stage)
pred = Prediction.from_completions(completions, signature=signature)
@@ -138,6 +116,72 @@ class Predict(Parameter):
return f"{self.__class__.__name__}({self.signature})"
+
+def old_generate(demos, signature, kwargs, config, lm, stage):
+ # Switch to legacy format for dsp.generate
+ x = dsp.Example(demos=demos, **kwargs)
+ template = signature_to_template(signature)
+
+ if lm is None:
+ x, C = dsp.generate(template, **config)(x, stage=stage)
+ else:
+ # Note: query_only=True means the instructions and examples are not included.
+ with dsp.settings.context(lm=lm, query_only=True):
+ x, C = dsp.generate(template, **config)(x, stage=stage)
+
+ # assert stage in x, "The generated (input, output) example was not stored"
+
+ completions = []
+
+ for c in C:
+ completions.append({})
+ for field in template.fields:
+ if field.output_variable not in kwargs.keys():
+ completions[-1][field.output_variable] = getattr(
+ c,
+ field.output_variable,
+ )
+
+ return completions
+
+
+def new_generate(lm, signature, example, max_depth=6, **kwargs):
+ # Generate and extract the fields.
+ template = signature_to_template(signature, adapter=dsp.ExperimentalAdapter)
+ prompt = template(example)
+ completions = lm(prompt, **kwargs)
+ completions = [template.extract(example, p) for p in completions]
+
+ assert all(set(signature.input_fields).issubset(set(c.keys())) for c in completions), "Missing input keys."
+
+ # Find the completions that are most complete.
+ field_names = [field.input_variable for field in template.fields]
+ for field_idx, key in enumerate(field_names):
+ completions_ = [c for c in completions if key in c.keys() and c[key] is not None]
+ completions = completions_ or completions
+ if len(completions_) == 0: break
+
+ # If none of the completions is completed (i.e., none has the final field set).
+ if len(completions_) == 0:
+ # Pick the first completion that has gone farthest.
+ completion = completions[0]
+
+ for field_idx_ in range(field_idx+1, len(field_names)):
+ if field_names[field_idx_] in completion: del completion[field_names[field_idx_]]
+
+ # Recurse with greedy decoding.
+ new_kwargs = {**kwargs, "n": 1, "temperature": 0.0,}
+
+ assert max_depth > 0
+ return new_generate(lm, signature, completion, max_depth=max_depth-1, **new_kwargs)
+
+ # Keep only output fields.
+ completions = [{k: v for k, v in c.items() if k in signature.output_fields} for c in completions]
+
+ return completions
+
+
+
# TODO: get some defaults during init from the context window?
# # TODO: FIXME: Hmm, I guess expected behavior is that contexts can
# affect execution. Well, we need to determine whether context dominates, __init__ demoninates, or forward dominates.
diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.py
index 77b51e622..228a0a552 100644
--- a/dspy/signatures/signature.py
+++ b/dspy/signatures/signature.py
@@ -14,9 +14,12 @@ import dsp
from dspy.signatures.field import InputField, OutputField, new_to_old_field
-def signature_to_template(signature) -> dsp.Template:
+def signature_to_template(signature, adapter=None) -> dsp.Template:
"""Convert from new to legacy format."""
- return dsp.Template(
+
+ adapter = adapter or dsp.Template
+
+ return adapter(
signature.instructions,
**{name: new_to_old_field(field) for name, field in signature.fields.items()},
)