Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Update modules. Include generic ReAct, MultiChainComparison, ChainOfThoughtWithHint, improve dspy.majority, tweak teleprompters, update default OpenAI LM to turbo-instruct

Omar Khattab680e00fb73a8
diff --git a/dsp/modules/gpt3.py b/dsp/modules/gpt3.py
index ab7ddf87c..2e91a51bc 100644
--- a/dsp/modules/gpt3.py
+++ b/dsp/modules/gpt3.py
@@ -33,7 +33,7 @@ class GPT3(LM):
def __init__(
self,
- model: str = "text-davinci-002",
+ model: str = "gpt-3.5-turbo-instruct",
api_key: Optional[str] = None,
api_provider: Literal["openai", "azure"] = "openai",
model_type: Literal["chat", "text"] = None,
@@ -42,7 +42,7 @@ class GPT3(LM):
super().__init__(model)
self.provider = "openai"
- default_model_type = "chat" if 'gpt-3.5' in model or 'gpt-4' in model else "text"
+ default_model_type = "chat" if ('gpt-3.5' in model or 'gpt-4' in model) and ('instruct' not in model) else "text"
self.model_type = model_type if model_type else default_model_type
if api_provider == "azure":
@@ -143,11 +143,11 @@ class GPT3(LM):
assert only_completed, "for now"
assert return_sorted is False, "for now"
- if kwargs.get("n", 1) > 1:
- if self.model_type == "chat":
- kwargs = {**kwargs}
- else:
- kwargs = {**kwargs, "logprobs": 5}
+ # if kwargs.get("n", 1) > 1:
+ # if self.model_type == "chat":
+ # kwargs = {**kwargs}
+ # else:
+ # kwargs = {**kwargs, "logprobs": 5}
response = self.request(prompt, **kwargs)
choices = response["choices"]
diff --git a/dsp/modules/hf.py b/dsp/modules/hf.py
index 13ef0499f..835524488 100644
--- a/dsp/modules/hf.py
+++ b/dsp/modules/hf.py
@@ -1,8 +1,8 @@
-from typing import Optional, Literal
import os
import json
# from peft import PeftConfig, PeftModel
# from transformers import AutoModelForSeq2SeqLM, AutoModelForCausalLM, AutoTokenizer, AutoConfig
+from typing import Optional, Literal
from dsp.modules.lm import LM
# from dsp.modules.finetuning.finetune_hf import preprocess_prompt
@@ -104,13 +104,15 @@ class HFModel(LM):
assert not self.is_client
# TODO: Add caching
kwargs = {**openai_to_hf(**self.kwargs), **openai_to_hf(**kwargs)}
- print(prompt)
+ # print(prompt)
if isinstance(prompt, dict):
try:
prompt = prompt['messages'][0]['content']
except (KeyError, IndexError, TypeError):
print("Failed to extract 'content' from the prompt.")
inputs = self.tokenizer(prompt, return_tensors="pt").to(self.device)
+
+ # print(kwargs)
outputs = self.model.generate(**inputs, **kwargs)
if self.drop_prompt_from_output:
input_length = inputs.input_ids.shape[1]
@@ -129,7 +131,7 @@ class HFModel(LM):
assert only_completed, "for now"
assert return_sorted is False, "for now"
- if kwargs.get("n", 1) > 1:
+ if kwargs.get("n", 1) > 1 or kwargs.get("temperature", 0.0) > 0.1:
kwargs["do_sample"] = True
response = self.request(prompt, **kwargs)
diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.py
index e368800a6..d8677ece4 100644
--- a/dsp/modules/hf_client.py
+++ b/dsp/modules/hf_client.py
@@ -1,26 +1,4 @@
-# import requests
-
-# from dsp.modules.hf import HFModel
-
-
-# class HFModelClient(HFModel):
-# def __init__(self, port, model, url="http://0.0.0.0"):
-# super().__init__(model=model, is_client=True)
-# self.url = f"{url}:{port}"
-# self.headers = {"Content-Type": "application/json; charset=utf-8"}
-
-# def _generate(self, prompt, **kwargs):
-# payload = {"prompt": prompt, **kwargs}
-# response = requests.post(self.url, json=payload, headers=self.headers)
-# try:
-# return response.json()
-# except:
-# print("Failed to parse JSON response:", response.text)
-# raise Exception("Received invalid JSON response from server")
-
-
-import dsp
-
+import random
import requests
from dsp.modules.hf import HFModel, openai_to_hf
from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_on
@@ -31,11 +9,14 @@ from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn
class HFClientTGI(HFModel):
def __init__(self, model, port, url="http://future-hgx-1", **kwargs):
super().__init__(model=model, is_client=True)
- self.url = f"{url}:{port}"
+
+ self.url = url
+ self.ports = port if isinstance(port, list) else [port]
+
self.headers = {"Content-Type": "application/json"}
self.kwargs = {
- "temperature": 0.1,
+ "temperature": 0.01,
"max_tokens": 75,
"top_p": 0.97,
"n": 1,
@@ -43,19 +24,21 @@ class HFClientTGI(HFModel):
**kwargs,
}
+ # print(self.kwargs)
+
def _generate(self, prompt, **kwargs):
kwargs = {**self.kwargs, **kwargs}
payload = {
- "inputs": prompt,
- "parameters": {
- "do_sample": kwargs["n"] > 1,
- "best_of": kwargs["n"],
- "details": kwargs["n"] > 1,
- # "max_new_tokens": kwargs.get('max_tokens', kwargs.get('max_new_tokens', 75)),
- "stop": ["\n", "\n\n"],
- **kwargs,
- },
+ "inputs": prompt,
+ "parameters": {
+ "do_sample": kwargs["n"] > 1,
+ "best_of": kwargs["n"],
+ "details": kwargs["n"] > 1,
+ # "max_new_tokens": kwargs.get('max_tokens', kwargs.get('max_new_tokens', 75)),
+ # "stop": ["\n", "\n\n"],
+ **kwargs,
+ }
}
payload["parameters"] = openai_to_hf(**payload["parameters"])
@@ -67,9 +50,8 @@ class HFClientTGI(HFModel):
# print(payload['parameters'])
# response = requests.post(self.url + "/generate", json=payload, headers=self.headers)
- response = send_hftgi_request_v00(
- self.url + "/generate", json=payload, headers=self.headers
- )
+
+ response = send_hftgi_request_v01(f"{self.url}:{random.Random().choice(self.ports)}" + "/generate", url=self.url, ports=tuple(self.ports), json=payload, headers=self.headers)
try:
json_response = response.json()
@@ -93,6 +75,10 @@ class HFClientTGI(HFModel):
raise Exception("Received invalid JSON response from server")
[email protected](ignore=['arg'])
+def send_hftgi_request_v01(arg, url, ports, **kwargs):
+ return requests.post(arg, **kwargs)
+
@CacheMemory.cache
def send_hftgi_request_v00(arg, **kwargs):
return requests.post(arg, **kwargs)
diff --git a/dsp/templates/template_v2.py b/dsp/templates/template_v2.py
index 5d42aec62..5bd751eee 100644
--- a/dsp/templates/template_v2.py
+++ b/dsp/templates/template_v2.py
@@ -154,6 +154,8 @@ class TemplateV2:
break
idx += 1
+ import dspy
+
idx = min(idx, len(self.fields) - 1)
while raw_pred != "" and idx < len(self.fields):
if idx < len(self.fields) - 1:
@@ -161,20 +163,32 @@ class TemplateV2:
offset = raw_pred.find(next_field_name)
if offset >= 0:
- example[self.fields[idx].output_variable] = raw_pred[
- :offset
- ].strip()
- raw_pred = raw_pred[offset + len(next_field_name) :].strip()
+ if dspy.settings.release >= 20231003:
+ example[self.fields[idx].output_variable] = raw_pred[:offset].strip().rstrip('---').strip()
+ raw_pred = raw_pred[offset + len(next_field_name) :].strip().rstrip('---').strip()
+ else:
+ example[self.fields[idx].output_variable] = raw_pred[:offset].strip()
+ raw_pred = raw_pred[offset + len(next_field_name) :].strip()
+
idx += 1
else:
- example[self.fields[idx].output_variable] = raw_pred.strip()
+ if dspy.settings.release >= 20231003:
+ example[self.fields[idx].output_variable] = raw_pred.strip().rstrip('---').strip()
+ else:
+ example[self.fields[idx].output_variable] = raw_pred.strip()
+
raw_pred = ""
idx += 1
break
else:
assert idx == len(self.fields) - 1, (idx, len(self.fields))
- example[self.fields[idx].output_variable] = raw_pred.strip()
+
+ if dspy.settings.release >= 20231003:
+ example[self.fields[idx].output_variable] = raw_pred.strip().rstrip('---').strip()
+ else:
+ example[self.fields[idx].output_variable] = raw_pred.strip()
+
break
return example
@@ -209,16 +223,23 @@ class TemplateV2:
# Move the rdemos to ademos if rdemo has all the fields filled in
rdemos_ = []
+ new_ademos = []
for rdemo in rdemos:
if all(
- field.name in rdemo
+ (field.name in rdemo)
for field in self.fields
if field.input_variable in example
):
- ademos.append(rdemo)
+ import dspy
+
+ if dspy.settings.release >= 20230928:
+ new_ademos.append(rdemo)
+ else:
+ ademos.append(rdemo)
else:
rdemos_.append(rdemo)
+ ademos = new_ademos + ademos
rdemos = rdemos_
diff --git a/dsp/templates/template_v3.py b/dsp/templates/template_v3.py
index 4ae0a9c3e..fb097f877 100644
--- a/dsp/templates/template_v3.py
+++ b/dsp/templates/template_v3.py
@@ -28,6 +28,7 @@ class Template(TemplateV2):
self.fields: list[Field] = []
self.format_handlers: dict[str, Callable] = {
"context": passages2text,
+ "passages": passages2text,
"answers": format_answers,
}
@@ -63,3 +64,10 @@ class Template(TemplateV2):
# print("here?", self.instructions == other.instructions, self.kwargs == other.kwargs)
return self.instructions == other.instructions and self.kwargs == other.kwargs
+
+ def __str__(self) -> str:
+ # field names
+ field_names = [field.name for field in self.fields]
+
+ return f"Template({self.instructions}, {field_names})"
+
diff --git a/dsp/utils/settings.py b/dsp/utils/settings.py
index 3d27eb384..120154fdd 100644
--- a/dsp/utils/settings.py
+++ b/dsp/utils/settings.py
@@ -31,7 +31,8 @@ class Settings(object):
force_reuse_cached_compilation=False,
compiling=False,
skip_logprobs=False,
- trace=None
+ trace=None,
+ release=0,
)
cls._instance.__append(config)
diff --git a/dspy/datasets/hotpotqa.py b/dspy/datasets/hotpotqa.py
index e1e0256ba..5d68a0689 100644
--- a/dspy/datasets/hotpotqa.py
+++ b/dspy/datasets/hotpotqa.py
@@ -49,6 +49,9 @@ class HotPotQA(Dataset):
for raw_example in hf_official_dev:
assert raw_example['level'] == 'hard'
example = {k: raw_example[k] for k in ['id', 'question', 'answer', 'type', 'supporting_facts']}
+ if 'supporting_facts' in example:
+ example['gold_titles'] = set(example['supporting_facts']['title'])
+ del example['supporting_facts']
test.append(example)
self._test = test
diff --git a/dspy/predict/__init__.py b/dspy/predict/__init__.py
index bda4e41a2..7683a2f30 100644
--- a/dspy/predict/__init__.py
+++ b/dspy/predict/__init__.py
@@ -1,3 +1,6 @@
from .predict import Predict
from .chain_of_thought import ChainOfThought
-from .aggregation import majority
\ No newline at end of file
+from .multi_chain_comparison import MultiChainComparison
+from .chain_of_thought_with_hint import ChainOfThoughtWithHint
+from .react import ReAct
+from .aggregation import majority
diff --git a/dspy/predict/aggregation.py b/dspy/predict/aggregation.py
index 4b44675e1..2212900c2 100644
--- a/dspy/predict/aggregation.py
+++ b/dspy/predict/aggregation.py
@@ -9,9 +9,10 @@ def majority(prediction_or_completions, normalize=default_normalize, field=None)
"""
Returns the most common completion for the target field (or the last field) in the signature.
When normalize returns None, that completion is ignored.
+ In case of a tie, earlier completion are prioritized.
"""
- assert isinstance(prediction_or_completions, Prediction) or isinstance(prediction_or_completions, Completions)
+ assert any(isinstance(prediction_or_completions, t) for t in [Prediction, Completions, list])
input_type = type(prediction_or_completions)
# Get the completions
@@ -20,8 +21,17 @@ def majority(prediction_or_completions, normalize=default_normalize, field=None)
else:
completions = prediction_or_completions
+ try:
+ signature = completions.signature
+ except:
+ signature = None
+
+ try:
+ field = field if field else signature.fields[-1].output_variable
+ except:
+ field = field if field else list(completions[0].keys())[-1]
+
# Normalize
- field = field if field else completions.signature.fields[-1].output_variable
normalize = normalize if normalize else lambda x: x
normalized_values = [normalize(completion[field]) for completion in completions]
normalized_values_ = [x for x in normalized_values if x is not None]
@@ -38,8 +48,8 @@ def majority(prediction_or_completions, normalize=default_normalize, field=None)
if normalize(completion[field]) == majority_value:
break
- if input_type == Prediction:
- return Prediction.from_completions([completion], signature=completions.signature)
+ # if input_type == Prediction:
+ return Prediction.from_completions([completion], signature=signature)
- return Completions([completion], signature=completions.signature)
+ return Completions([completion], signature=signature)
diff --git a/dspy/predict/chain_of_thought.py b/dspy/predict/chain_of_thought.py
index d41003c8d..4943856b9 100644
--- a/dspy/predict/chain_of_thought.py
+++ b/dspy/predict/chain_of_thought.py
@@ -6,6 +6,27 @@ from .predict import Predict
# TODO: FIXME: Insert this right before the *first* output field. Also rewrite this to use the new signature system.
+# TODO: This shouldn't inherit from Predict. It should be a module that has one or two predictors.
+# Let's focus on the activated case. It's a predictor with the expanded signature.
+# Now, when deactivated, it's a predictor with the original signature.
+# When activate is None, though, we need the expanded one but during forward we need to pass the right signature.
+
+"""
+class ChainOfThought(dspy.Module):
+ def __init__(self, signature):
+
+ input_fields, output_fields = dspy.process_signature(signature)
+ output_fields = dict(rationale=dspy.OutputField(prefix="Reasoning: Let's think step by step."), **output_fields)
+ self.signature = dspy.Signature(input_fields, output_fields)
+
+ self.predict = dspy.Predict(self.signature)
+
+ def forward(self, **kwargs):
+ return self.predict(**kwargs)
+
+# How this should look like. But with also passing signature=simpler_signature to the predict module *if* deactivated.
+"""
+
class ChainOfThought(Predict):
def __init__(self, signature, rationale_type=None, activated=True, **config):
super().__init__(signature, **config)
@@ -38,4 +59,4 @@ class ChainOfThought(Predict):
TODO: In principle, we can update the field's prefix during forward too to fill any thing based on the input args.
IF the user didn't overwrite our default rationale_type.
-"""
\ No newline at end of file
+"""
diff --git a/dspy/predict/chain_of_thought_with_hint.py b/dspy/predict/chain_of_thought_with_hint.py
new file mode 100644
index 000000000..b968d0bd9
--- /dev/null
+++ b/dspy/predict/chain_of_thought_with_hint.py
@@ -0,0 +1,50 @@
+import dsp
+
+from .predict import Predict
+
+
+
+# TODO: FIXME: Insert this right before the *first* output field. Also rewrite this to use the new signature system.
+
+class ChainOfThoughtWithHint(Predict):
+ def __init__(self, signature, rationale_type=None, activated=True, **config):
+ super().__init__(signature, **config)
+
+ self.activated = activated
+
+ signature = self.signature
+ *keys, last_key = signature.kwargs.keys()
+
+ DEFAULT_HINT_TYPE = dsp.Type(prefix="Hint:", desc="${hint}")
+
+ DEFAULT_RATIONALE_TYPE = dsp.Type(prefix="Reasoning: Let's think step by step in order to",
+ desc="${produce the " + last_key + "}. We ...")
+
+ rationale_type = rationale_type or DEFAULT_RATIONALE_TYPE
+
+ extended_kwargs1 = {key: signature.kwargs[key] for key in keys}
+ extended_kwargs1.update({'rationale': rationale_type, last_key: signature.kwargs[last_key]})
+
+ extended_kwargs2 = {key: signature.kwargs[key] for key in keys}
+ extended_kwargs2.update({'hint': DEFAULT_HINT_TYPE, 'rationale': rationale_type, last_key: signature.kwargs[last_key]})
+
+ self.extended_signature1 = dsp.Template(signature.instructions, **extended_kwargs1)
+ self.extended_signature2 = dsp.Template(signature.instructions, **extended_kwargs2)
+
+ def forward(self, **kwargs):
+ signature = self.signature
+
+ if self.activated is True or (self.activated is None and isinstance(dsp.settings.lm, dsp.GPT3)):
+ if 'hint' in kwargs and kwargs['hint']:
+ signature = self.extended_signature2
+ else:
+ signature = self.extended_signature1
+
+ return super().forward(signature=signature, **kwargs)
+
+
+"""
+TODO: In principle, we can update the field's prefix during forward too to fill any thing based on the input args.
+
+IF the user didn't overwrite our default rationale_type.
+"""
\ No newline at end of file
diff --git a/dspy/predict/multi_chain_comparison.py b/dspy/predict/multi_chain_comparison.py
new file mode 100644
index 000000000..99c2b43c5
--- /dev/null
+++ b/dspy/predict/multi_chain_comparison.py
@@ -0,0 +1,38 @@
+from .predict import Predict
+from ..primitives.program import Module
+
+import dsp
+
+class MultiChainComparison(Module):
+ def __init__(self, signature, M=3, temperature=0.7, **config):
+ super().__init__()
+
+ self.M = M
+ signature = Predict(signature).signature
+ *keys, last_key = signature.kwargs.keys()
+
+ extended_kwargs = {key: signature.kwargs[key] for key in keys}
+
+ for idx in range(M):
+ candidate_type = dsp.Type(prefix=f"Student Attempt #{idx+1}:", desc="${reasoning attempt}")
+ extended_kwargs.update({f'reasoning_attempt_{idx+1}': candidate_type})
+
+ rationale_type = dsp.Type(prefix="Accurate Reasoning: Thank you everyone. Let's now holistically", desc="${corrected reasoning}")
+ extended_kwargs.update({'rationale': rationale_type, last_key: signature.kwargs[last_key]})
+
+ signature = dsp.Template(signature.instructions, **extended_kwargs)
+ self.predict = Predict(signature, temperature=temperature, **config)
+ self.last_key = last_key
+
+ def forward(self, completions, **kwargs):
+ attempts = []
+
+ for c in completions:
+ rationale = c.rationale.strip().split('\n')[0].strip()
+ answer = c[self.last_key].strip().split('\n')[0].strip()
+ attempts.append(f"«I'm trying to {rationale} I'm not sure but my prediction is {answer}»")
+
+ assert len(attempts) == self.M, len(attempts)
+
+ kwargs = {**{f'reasoning_attempt_{idx+1}': attempt for idx, attempt in enumerate(attempts)}, **kwargs}
+ return self.predict(**kwargs)
diff --git a/dspy/predict/react.py b/dspy/predict/react.py
new file mode 100644
index 000000000..b31e4349d
--- /dev/null
+++ b/dspy/predict/react.py
@@ -0,0 +1,84 @@
+import dsp
+import dspy
+from ..primitives.program import Module
+from .predict import Predict
+
+# TODO: Simplify a lot.
+# TODO: Divide Action and Action Input like langchain does for ReAct.
+
+
+class ReAct(Module):
+ def __init__(self, signature, max_iters=5, num_results=3, tools=None):
+ super().__init__()
+ self.signature = signature = dspy.Predict(signature).signature
+ self.max_iters = max_iters
+
+ self.tools = tools or [dspy.Retrieve(k=num_results)]
+ self.tools = {tool.name: tool for tool in self.tools} #if isinstance(self.tools, list) else self.tools
+
+ self.input_fields = {k: v for k, v in self.signature.kwargs.items() if isinstance(v, dspy.InputField)}
+ self.output_fields = {k: v for k, v in self.signature.kwargs.items() if isinstance(v, dspy.OutputField)}
+
+ inputs, outputs = signature.fields[:-1], signature.fields[-1:]
+
+ inputs_ = ', '.join([f"`{field.input_variable}`" for field in inputs])
+ outputs_ = ', '.join([f"`{field.output_variable}`" for field in outputs])
+
+ assert len(outputs) == 1, "ReAct only supports one output field."
+
+ instr = []
+ instr.append(f"You will be given {inputs_} and you will respond with {outputs_}.\n")
+ instr.append("To do this, you will interleave Thought, Action, and Observation steps.\n")
+ instr.append("Thought can reason about the current situation, and Action can be the following types:\n")
+
+ self.tools['Finish'] = dspy.Example(name="Finish", input_variable=outputs_.strip('`'), desc=f"returns the final {outputs_} and finishes the task")
+
+ for idx, tool in enumerate(self.tools):
+ tool = self.tools[tool]
+ instr.append(f"({idx+1}) {tool.name}[{tool.input_variable}], which {tool.desc}")
+
+ instr = '\n'.join(instr)
+ self.react = [Predict(dsp.Template(instr, **self._generate_signature(i))) for i in range(1, max_iters + 1)]
+
+ def _generate_signature(self, iters):
+ signature_dict = {}
+ for key, val in self.input_fields.items():
+ signature_dict[key] = val
+
+ for j in range(1, iters + 1):
+ signature_dict[f"Thought_{j}"] = dspy.OutputField(prefix=f"Thought {j}:", desc="next steps to take based on last observation")
+
+ tool_list = 'or '.join([f"{tool.name}[{tool.input_variable}]" for tool in self.tools.values() if tool.name != 'Finish'])
+ signature_dict[f"Action_{j}"] = dspy.OutputField(prefix=f"Action {j}:", desc=f"always either {tool_list} or, when done, Finish[answer]")
+
+ if j < iters:
+ signature_dict[f"Observation_{j}"] = dspy.OutputField(prefix=f"Observation {j}:", desc="observations based on action", format=dsp.passages2text)
+
+ return signature_dict
+
+ def act(self, output, hop):
+ try:
+ action = output[f"Action_{hop+1}"]
+ action_name, action_val = action.strip().split('\n')[0].split('[', 1)
+ action_val = action_val.rsplit(']', 1)[0]
+
+ if action_name == 'Finish': return action_val
+
+ output[f"Observation_{hop+1}"] = self.tools[action_name](action_val).passages
+
+ except Exception as e:
+ output[f"Observation_{hop+1}"] = "Failed to parse action. Bad formatting or incorrect action name."
+
+
+ def forward(self, **kwargs):
+ args = {key: kwargs[key] for key in self.input_fields.keys() if key in kwargs}
+
+ for hop in range(self.max_iters):
+ # with dspy.settings.context(show_guidelines=(i <= 2)):
+ output = self.react[hop](**args)
+
+ if action_val := self.act(output, hop): break
+ args.update(output)
+
+ # assumes only 1 output field for now - TODO: handling for multiple output fields
+ return dspy.Prediction(**{list(self.output_fields.keys())[0]: action_val or ''})
diff --git a/dspy/primitives/module.py b/dspy/primitives/module.py
index 2ad125d8f..866fb2009 100644
--- a/dspy/primitives/module.py
+++ b/dspy/primitives/module.py
@@ -26,8 +26,10 @@ class BaseModule:
add_parameter(name, value)
elif isinstance(value, BaseModule):
- for sub_name, param in value.named_parameters():
- add_parameter(f"{name}.{sub_name}", param)
+ # When a sub-module is pre-compiled, keep it frozen.
+ if not value._compiled:
+ for sub_name, param in value.named_parameters():
+ add_parameter(f"{name}.{sub_name}", param)
elif isinstance(value, (list, tuple)):
for idx, item in enumerate(value):
diff --git a/dspy/retrieve/retrieve.py b/dspy/retrieve/retrieve.py
index 0d62cdbba..4f61ba8ae 100644
--- a/dspy/retrieve/retrieve.py
+++ b/dspy/retrieve/retrieve.py
@@ -6,6 +6,10 @@ from dspy.primitives.prediction import Prediction
class Retrieve(Parameter):
+ name = "Search"
+ input_variable = "query"
+ desc = "takes a search query and returns one or more potentially relevant passages from a corpus"
+
def __init__(self, k=3):
self.stage = random.randbytes(8).hex()
self.k = k
@@ -26,6 +30,11 @@ class Retrieve(Parameter):
def forward(self, query_or_queries):
queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries
+ queries = [query.strip().split('\n')[0].strip() for query in queries]
+
+
+ # print(queries)
+ # TODO: Consider removing any quote-like markers that surround the query too.
passages = dsp.retrieveEnsemble(queries, k=self.k)
return Prediction(passages=passages)
diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.py
index f742addb8..098acfd02 100644
--- a/dspy/signatures/signature.py
+++ b/dspy/signatures/signature.py
@@ -48,8 +48,10 @@ class SignatureMeta(type):
return f'{cls.__name__}\n' + '\n'.join(s)
+
class Signature(metaclass=SignatureMeta):
- pass
+ def __eq__(self, __value: object) -> bool:
+ return self._template == __value._template
diff --git a/dspy/teleprompt/__init__.py b/dspy/teleprompt/__init__.py
index f784f6266..b4543a8e6 100644
--- a/dspy/teleprompt/__init__.py
+++ b/dspy/teleprompt/__init__.py
@@ -2,4 +2,5 @@ from .teleprompt import *
from .bootstrap import *
from .vanilla import *
from .random_search import *
-from .finetune import *
\ No newline at end of file
+from .finetune import *
+
diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.py
index 9f42ff9a7..14588592d 100644
--- a/dspy/teleprompt/bootstrap.py
+++ b/dspy/teleprompt/bootstrap.py
@@ -16,14 +16,15 @@ from dspy.evaluate.evaluate import Evaluate
# NOTE: Notice the places where we don't shuffle examples. I do like that this one doesn't shuffle.
# Other ones that consider options may want to use both unshuffled and then shuffle a few times, when considering candidates.
-# TODO: the max_rounds should be supported here. First round is the same way it is now: minimal intervention.
-# Past that first round, though, we can set the temperature to be high and we need to use branch_idx to
-# get past the cache. In principle, we can also sample multiple outputs from the final generation step
+# TODO: the max_rounds via branch_idx to get past the cache, not just temperature.
+# In principle, we can also sample multiple outputs from the final generation step
# (or even each step, in case the validation function just wants *one* thing that works, but nah)
# and try them all. Having a pretty solid guess on the "final step" of each example isn't hard by the second round,
# in the sense that we have the trace from the first round. (Yes it may change but that's an edge case that
# won't hurt our "best effort" guarantees.)
+# TODO: When this bootstraps for another teleprompter like finetune, we want all demos we gather.
+# But when it's for direct use we may want to sample ONE demo per predictor--example pair. This is important for "multi-use" modules.
# TODO: Add baselines=[...]
@@ -64,9 +65,11 @@ class BootstrapFewShot(Teleprompter):
name2predictor, predictor2name = {}, {}
student, teacher = self.student, self.teacher
+ assert len(student.predictors()) == len(teacher.predictors()), "Student and teacher must have the same number of predictors."
+
for (name1, predictor1), (name2, predictor2) in zip(student.named_predictors(), teacher.named_predictors()):
assert name1 == name2, "Student and teacher must have the same program structure."
- assert predictor1.signature == predictor2.signature, "Student and teacher must have the same signatures."
+ assert predictor1.signature == predictor2.signature, f"Student and teacher must have the same signatures. {type(predictor1.signature)} != {type(predictor2.signature)}"
assert id(predictor1) != id(predictor2), "Student and teacher must be different objects."
name2predictor[name1] = None # dict(student=predictor1, teacher=predictor2)
@@ -111,28 +114,29 @@ class BootstrapFewShot(Teleprompter):
teacher = self.teacher #.deepcopy()
predictor_cache = {}
- with dsp.settings.context(trace=[], **self.teacher_settings):
- lm = dsp.settings.lm
- lm = lm.copy(temperature=0.7 + 0.001 * round_idx) if round_idx > 0 else lm
- new_settings = dict(lm=lm) if round_idx > 0 else {}
+ try:
+ with dsp.settings.context(trace=[], **self.teacher_settings):
+ lm = dsp.settings.lm
+ lm = lm.copy(temperature=0.7 + 0.001 * round_idx) if round_idx > 0 else lm
+ new_settings = dict(lm=lm) if round_idx > 0 else {}
- with dsp.settings.context(**new_settings):
- for name, predictor in teacher.named_predictors():
- predictor_cache[name] = predictor.demos
- predictor.demos = [x for x in predictor.demos if x != example]
+ with dsp.settings.context(**new_settings):
+ for name, predictor in teacher.named_predictors():
+ predictor_cache[name] = predictor.demos
+ predictor.demos = [x for x in predictor.demos if x != example]
- prediction = teacher(**example.inputs())
- trace = dsp.settings.trace
+ prediction = teacher(**example.inputs())
+ trace = dsp.settings.trace
- for name, predictor in teacher.named_predictors():
- predictor.demos = predictor_cache[name]
+ for name, predictor in teacher.named_predictors():
+ predictor.demos = predictor_cache[name]
- try:
- success = (self.metric is None) or self.metric(example, prediction, trace)
- # print(success, example, prediction)
+ success = (self.metric is None) or self.metric(example, prediction, trace)
+ # print(success, example, prediction)
except Exception as e:
success = False
- print(f'Failed to evaluate example {example.dspy_uuid} with {self.metric} due to {e}.')
+ # FIXME: remove the reliance on uuid here so the error is printed
+ print(f'Failed to run or to evaluate example {example} with {self.metric} due to {e}.')
if success:
for step in trace:
@@ -147,6 +151,8 @@ class BootstrapFewShot(Teleprompter):
try:
predictor_name = self.predictor2name[id(predictor)]
except KeyError as e:
+ continue # FIXME: !
+
# TODO: Look closer into this. It's a bit tricky to reproduce.
print(f'Failed to find predictor {predictor} in {self.predictor2name}.')
print('Are you doing this in a notebook (Jupyter)? This might be caused by redefining values by rerunning cells.')
@@ -168,6 +174,11 @@ class BootstrapFewShot(Teleprompter):
sample_size = max(0, sample_size)
raw_demos = rng.sample(raw_demos, sample_size)
- predictor.demos = augmented_demos + raw_demos
+
+ import dspy
+ if dspy.settings.release >= 20230928:
+ predictor.demos = raw_demos + augmented_demos
+ else:
+ predictor.demos = augmented_demos + raw_demos
return self.student
diff --git a/dspy/teleprompt/ensemble.py b/dspy/teleprompt/ensemble.py
new file mode 100644
index 000000000..f52dc1021
--- /dev/null
+++ b/dspy/teleprompt/ensemble.py
@@ -0,0 +1,40 @@
+import dsp
+import tqdm
+import random
+
+from dspy.teleprompt.teleprompt import Teleprompter
+
+"""
+TODO: The EnsembledProgram should actually imitate the structure of the individual programs (IF they are all compatible). This allows compiling with an ensemble program as a (singular) teacher. Basically the top majority-compatible trace will end up being used, if dspy.majority is the reduce_fn.
+"""
+
+class Ensemble(Teleprompter):
+ def __init__(self, *, reduce_fn=None, size=None, deterministic=False):
+ """A common reduce_fn is dspy.majority."""
+
+ assert deterministic is False, "TODO: Implement example hashing for deterministic ensemble."
+
+ self.reduce_fn = reduce_fn
+ self.size = size
+ self.deterministic = deterministic
+
+ def compile(self, programs):
+ size = self.size
+ reduce_fn = self.reduce_fn
+
+ import dspy
+ class EnsembledProgram(dspy.Module):
+ def __init__(self):
+ super().__init__()
+ self.programs = programs
+
+ def forward(self, *args, **kwargs):
+ programs = random.sample(self.programs, size) if size else self.programs
+ outputs = [prog(*args, **kwargs) for prog in programs]
+
+ if reduce_fn:
+ return reduce_fn(outputs)
+
+ return outputs
+
+ return EnsembledProgram()
diff --git a/dspy/teleprompt/finetune.py b/dspy/teleprompt/finetune.py
index 4c7e9571b..a84655bf7 100644
--- a/dspy/teleprompt/finetune.py
+++ b/dspy/teleprompt/finetune.py
@@ -54,37 +54,40 @@ class BootstrapFinetune(Teleprompter):
metric = metric or (lambda *args: True)
self.teleprompter = BootstrapFewShot(metric=metric,
max_bootstrapped_demos=999999,
- max_labeled_demos=0, # FIXME: TODO: Make this zero? or param, 16 by default?
+ max_labeled_demos=0, # FIXME: TODO: Make this zero? or param, with default as 16 or 0?
teacher_settings=teacher_settings)
def compile(self, student, *, teacher=None, trainset, valset=None,
- target='t5-large', bsize=12, accumsteps=1, lr=5e-5, epochs=1, bf16=False):
+ target='t5-large', bsize=12, accumsteps=1, lr=5e-5, epochs=1, bf16=False, int8=False, peft=False):
# It's usually better to supply a few-shot teacher, rather than uncompiled module (the student).
if teacher is None:
print("WARNING: Using a vanilla teacher. "
"Are you sure you want to use BootstrapFinetune without a compiled teacher?")
- # Dummy compilation to get bootstraps.
- compiled = self.teleprompter.compile(student, teacher=teacher, trainset=trainset)
- multitask = self.multitask
- #
- # Prepare finetune <prompt, completion> pairs.
- #
+ teachers = teacher if isinstance(teacher, list) else [teacher]
finetune_data = {}
- for name, predictor in compiled.named_predictors():
- name_ = 'all' if multitask else name
- finetune_data[name_] = [] if name_ not in finetune_data else finetune_data[name_]
+ for teacher in teachers:
+ # Dummy compilation to get bootstraps.
+ compiled = self.teleprompter.compile(student, teacher=teacher, trainset=trainset)
+ multitask = self.multitask
+
+ # Prepare finetune <prompt, completion> pairs.
+ for name, predictor in compiled.named_predictors():
+ name_ = 'all' if multitask else name
+ finetune_data[name_] = [] if name_ not in finetune_data else finetune_data[name_]
- for demo in predictor.demos:
- demo = dict(demo)
- completion = demo.pop(predictor.signature.fields[-1].output_variable)
- prompt = predictor.signature.query(dsp.Example(demos=[], **demo)).strip()
+ for demo in predictor.demos:
+ demo = dict(demo)
- finetune_data[name_].append(dict(prompt=prompt, completion=completion))
+ # TODO: FIXME: generalize.
+ completion = demo.pop(predictor.signature.fields[-1].output_variable)
+ prompt = predictor.signature.query(dsp.Example(demos=[], **demo)).strip()
+
+ finetune_data[name_].append(dict(prompt=prompt, completion=completion))
for name_ in finetune_data:
random.Random(0).shuffle(finetune_data[name_])
@@ -115,9 +118,10 @@ class BootstrapFinetune(Teleprompter):
import string
compiler_config = {
'save': ''.join(random.Random(time.time()).choices(string.ascii_uppercase + string.digits, k=13)), # https://stackoverflow.com/a/2257449/1493011
- 'peft': False,
+ 'peft': peft,
'fp16': False,
'bf16': bf16,
+ 'int8': int8,
'fid': False,
'rationale': False,
'batch_size': bsize,
@@ -135,9 +139,11 @@ class BootstrapFinetune(Teleprompter):
training_data_path = finetune_paths[name]
compiler_config_ = dict(compiler_config)
compiler_config_['save'] = compiler_config['save'] + '.' + name
- best_ckpt_path = finetune_hf(training_data_path, target, compiler_config_)
+ best_ckpt_path, tokenized_data = finetune_hf(training_data_path, target, compiler_config_)
+
+ print(f"#> Best checkpoint path: {best_ckpt_path} for {name}")
finetune_models[name] = dsp.HFModel(model=target, checkpoint=best_ckpt_path) # best_ckpt_path
-
+
#
# Set the LMs to the finetuned ones, per module
#
@@ -151,8 +157,9 @@ class BootstrapFinetune(Teleprompter):
# TODO: FIXME: When we assign .lm, the Predict.forward will also set only_query=True.
# This is correct for here but we may want to make it more explicitly restricted to finetuned models.
+ print(f"Assigning the LM of predictor {name}.")
+
predictor2.lm = finetune_models[name]
assert predictor2.demos == []
return compiled2
-
diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.py
index c2baf3785..f027f83fe 100644
--- a/dspy/teleprompt/random_search.py
+++ b/dspy/teleprompt/random_search.py
@@ -48,7 +48,7 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):
# print("Going to sample", self.max_num_traces, "traces in total.")
print("Will attempt to train", self.num_candidate_sets, "candidate sets.")
- def compile(self, student, *, teacher=None, trainset, valset=None):
+ def compile(self, student, *, teacher=None, trainset, valset=None, restrict=None):
self.trainset = trainset
self.valset = valset or trainset # TODO: FIXME: Note this choice.
@@ -57,6 +57,10 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):
score_data = []
for seed in range(-3, self.num_candidate_sets):
+ if (restrict is not None) and (seed not in restrict):
+ print(seed, restrict)
+ continue
+
trainset2 = list(self.trainset)
if seed == -3:
@@ -96,32 +100,38 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):
all_subscores.append(subscores)
print('Score:', score, 'for set:', [len(predictor.demos) for predictor in program2.predictors()])
- scores.append(score)
-
- print(f"Scores so far: {scores}")
- if score >= max(scores):
+ if len(scores) == 0 or score > max(scores):
print('New best score:', score, 'for seed', seed)
best_program = program2
+ scores.append(score)
+ print(f"Scores so far: {scores}")
+
print('Best score:', max(scores))
- score_data.append((score, subscores))
+ score_data.append((score, subscores, seed, program2))
if len(score_data) > 2: # We check if there are at least 3 scores to consider
for k in [1, 2, 3, 5, 8, 9999]:
top_3_scores = sorted(score_data, key=lambda x: x[0], reverse=True)[:k]
# Transpose the subscores to get max per entry and then calculate their average
- transposed_subscores = zip(*[subscores for _, subscores in top_3_scores if subscores])
+ transposed_subscores = zip(*[subscores for _, subscores, *_ in top_3_scores if subscores])
avg_of_max_per_entry = sum(max(entry) for entry in transposed_subscores) / len(top_3_scores[0][1])
print(f'Average of max per entry across top {k} scores: {avg_of_max_per_entry}')
-
+
if self.stop_at_score is not None and score >= self.stop_at_score:
print(f"Stopping early because score {score} is >= stop_at_score {self.stop_at_score}")
break
+ # To best program, attach all program candidates in decreasing average score
+ best_program.candidate_programs = score_data
+ best_program.candidate_programs = sorted(best_program.candidate_programs, key=lambda x: x[0], reverse=True)
+
+ print(len(best_program.candidate_programs), "candidate programs found.")
+
return best_program
diff --git a/dspy/teleprompt/vanilla.py b/dspy/teleprompt/vanilla.py
index 619733691..068343fa0 100644
--- a/dspy/teleprompt/vanilla.py
+++ b/dspy/teleprompt/vanilla.py
@@ -8,7 +8,7 @@ class LabeledFewShot(Teleprompter):
def __init__(self, k=16):
self.k = k
- def compile(self, student, *, trainset):
+ def compile(self, student, *, trainset, sample=True):
self.student = student.reset_copy()
self.trainset = trainset
@@ -18,7 +18,10 @@ class LabeledFewShot(Teleprompter):
rng = random.Random(0)
for predictor in self.student.predictors():
- predictor.demos = rng.sample(self.trainset, min(self.k, len(self.trainset)))
+ if sample:
+ predictor.demos = rng.sample(self.trainset, min(self.k, len(self.trainset)))
+ else:
+ predictor.demos = self.trainset[:min(self.k, len(self.trainset))]
return self.student