DSPy: The framework for programming—not prompting—language models
Python
Update modules. Include generic ReAct, MultiChainComparison, ChainOfThoughtWithHint, improve dspy.majority, tweak teleprompters, update default OpenAI LM to turbo-instruct
Omar Khattab680e00fb73a8
diff --git a/dsp/modules/gpt3.py b/dsp/modules/gpt3.pyindex ab7ddf87c..2e91a51bc 100644--- a/dsp/modules/gpt3.py+++ b/dsp/modules/gpt3.py@@ -33,7 +33,7 @@ class GPT3(LM):def __init__(self,- model: str = "text-davinci-002",+ model: str = "gpt-3.5-turbo-instruct",api_key: Optional[str] = None,api_provider: Literal["openai", "azure"] = "openai",model_type: Literal["chat", "text"] = None,@@ -42,7 +42,7 @@ class GPT3(LM):super().__init__(model)self.provider = "openai"- default_model_type = "chat" if 'gpt-3.5' in model or 'gpt-4' in model else "text"+ default_model_type = "chat" if ('gpt-3.5' in model or 'gpt-4' in model) and ('instruct' not in model) else "text"self.model_type = model_type if model_type else default_model_typeif api_provider == "azure":@@ -143,11 +143,11 @@ class GPT3(LM):assert only_completed, "for now"assert return_sorted is False, "for now"- if kwargs.get("n", 1) > 1:- if self.model_type == "chat":- kwargs = {**kwargs}- else:- kwargs = {**kwargs, "logprobs": 5}+ # if kwargs.get("n", 1) > 1:+ # if self.model_type == "chat":+ # kwargs = {**kwargs}+ # else:+ # kwargs = {**kwargs, "logprobs": 5}response = self.request(prompt, **kwargs)choices = response["choices"]diff --git a/dsp/modules/hf.py b/dsp/modules/hf.pyindex 13ef0499f..835524488 100644--- a/dsp/modules/hf.py+++ b/dsp/modules/hf.py@@ -1,8 +1,8 @@-from typing import Optional, Literalimport osimport json# from peft import PeftConfig, PeftModel# from transformers import AutoModelForSeq2SeqLM, AutoModelForCausalLM, AutoTokenizer, AutoConfig+from typing import Optional, Literalfrom dsp.modules.lm import LM# from dsp.modules.finetuning.finetune_hf import preprocess_prompt@@ -104,13 +104,15 @@ class HFModel(LM):assert not self.is_client# TODO: Add cachingkwargs = {**openai_to_hf(**self.kwargs), **openai_to_hf(**kwargs)}- print(prompt)+ # print(prompt)if isinstance(prompt, dict):try:prompt = prompt['messages'][0]['content']except (KeyError, IndexError, TypeError):print("Failed to extract 'content' from the prompt.")inputs = self.tokenizer(prompt, return_tensors="pt").to(self.device)++ # print(kwargs)outputs = self.model.generate(**inputs, **kwargs)if self.drop_prompt_from_output:input_length = inputs.input_ids.shape[1]@@ -129,7 +131,7 @@ class HFModel(LM):assert only_completed, "for now"assert return_sorted is False, "for now"- if kwargs.get("n", 1) > 1:+ if kwargs.get("n", 1) > 1 or kwargs.get("temperature", 0.0) > 0.1:kwargs["do_sample"] = Trueresponse = self.request(prompt, **kwargs)diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.pyindex e368800a6..d8677ece4 100644--- a/dsp/modules/hf_client.py+++ b/dsp/modules/hf_client.py@@ -1,26 +1,4 @@-# import requests--# from dsp.modules.hf import HFModel---# class HFModelClient(HFModel):-# def __init__(self, port, model, url="http://0.0.0.0"):-# super().__init__(model=model, is_client=True)-# self.url = f"{url}:{port}"-# self.headers = {"Content-Type": "application/json; charset=utf-8"}--# def _generate(self, prompt, **kwargs):-# payload = {"prompt": prompt, **kwargs}-# response = requests.post(self.url, json=payload, headers=self.headers)-# try:-# return response.json()-# except:-# print("Failed to parse JSON response:", response.text)-# raise Exception("Received invalid JSON response from server")---import dsp-+import randomimport requestsfrom dsp.modules.hf import HFModel, openai_to_hffrom dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_on@@ -31,11 +9,14 @@ from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turnclass HFClientTGI(HFModel):def __init__(self, model, port, url="http://future-hgx-1", **kwargs):super().__init__(model=model, is_client=True)- self.url = f"{url}:{port}"++ self.url = url+ self.ports = port if isinstance(port, list) else [port]+self.headers = {"Content-Type": "application/json"}self.kwargs = {- "temperature": 0.1,+ "temperature": 0.01,"max_tokens": 75,"top_p": 0.97,"n": 1,@@ -43,19 +24,21 @@ class HFClientTGI(HFModel):**kwargs,}+ # print(self.kwargs)+def _generate(self, prompt, **kwargs):kwargs = {**self.kwargs, **kwargs}payload = {- "inputs": prompt,- "parameters": {- "do_sample": kwargs["n"] > 1,- "best_of": kwargs["n"],- "details": kwargs["n"] > 1,- # "max_new_tokens": kwargs.get('max_tokens', kwargs.get('max_new_tokens', 75)),- "stop": ["\n", "\n\n"],- **kwargs,- },+ "inputs": prompt,+ "parameters": {+ "do_sample": kwargs["n"] > 1,+ "best_of": kwargs["n"],+ "details": kwargs["n"] > 1,+ # "max_new_tokens": kwargs.get('max_tokens', kwargs.get('max_new_tokens', 75)),+ # "stop": ["\n", "\n\n"],+ **kwargs,+ }}payload["parameters"] = openai_to_hf(**payload["parameters"])@@ -67,9 +50,8 @@ class HFClientTGI(HFModel):# print(payload['parameters'])# response = requests.post(self.url + "/generate", json=payload, headers=self.headers)- response = send_hftgi_request_v00(- self.url + "/generate", json=payload, headers=self.headers- )++ response = send_hftgi_request_v01(f"{self.url}:{random.Random().choice(self.ports)}" + "/generate", url=self.url, ports=tuple(self.ports), json=payload, headers=self.headers)try:json_response = response.json()@@ -93,6 +75,10 @@ class HFClientTGI(HFModel):raise Exception("Received invalid JSON response from server")[email protected](ignore=['arg'])+def send_hftgi_request_v01(arg, url, ports, **kwargs):+ return requests.post(arg, **kwargs)+@CacheMemory.cachedef send_hftgi_request_v00(arg, **kwargs):return requests.post(arg, **kwargs)diff --git a/dsp/templates/template_v2.py b/dsp/templates/template_v2.pyindex 5d42aec62..5bd751eee 100644--- a/dsp/templates/template_v2.py+++ b/dsp/templates/template_v2.py@@ -154,6 +154,8 @@ class TemplateV2:breakidx += 1+ import dspy+idx = min(idx, len(self.fields) - 1)while raw_pred != "" and idx < len(self.fields):if idx < len(self.fields) - 1:@@ -161,20 +163,32 @@ class TemplateV2:offset = raw_pred.find(next_field_name)if offset >= 0:- example[self.fields[idx].output_variable] = raw_pred[- :offset- ].strip()- raw_pred = raw_pred[offset + len(next_field_name) :].strip()+ if dspy.settings.release >= 20231003:+ example[self.fields[idx].output_variable] = raw_pred[:offset].strip().rstrip('---').strip()+ raw_pred = raw_pred[offset + len(next_field_name) :].strip().rstrip('---').strip()+ else:+ example[self.fields[idx].output_variable] = raw_pred[:offset].strip()+ raw_pred = raw_pred[offset + len(next_field_name) :].strip()+idx += 1else:- example[self.fields[idx].output_variable] = raw_pred.strip()+ if dspy.settings.release >= 20231003:+ example[self.fields[idx].output_variable] = raw_pred.strip().rstrip('---').strip()+ else:+ example[self.fields[idx].output_variable] = raw_pred.strip()+raw_pred = ""idx += 1breakelse:assert idx == len(self.fields) - 1, (idx, len(self.fields))- example[self.fields[idx].output_variable] = raw_pred.strip()++ if dspy.settings.release >= 20231003:+ example[self.fields[idx].output_variable] = raw_pred.strip().rstrip('---').strip()+ else:+ example[self.fields[idx].output_variable] = raw_pred.strip()+breakreturn example@@ -209,16 +223,23 @@ class TemplateV2:# Move the rdemos to ademos if rdemo has all the fields filled inrdemos_ = []+ new_ademos = []for rdemo in rdemos:if all(- field.name in rdemo+ (field.name in rdemo)for field in self.fieldsif field.input_variable in example):- ademos.append(rdemo)+ import dspy++ if dspy.settings.release >= 20230928:+ new_ademos.append(rdemo)+ else:+ ademos.append(rdemo)else:rdemos_.append(rdemo)+ ademos = new_ademos + ademosrdemos = rdemos_diff --git a/dsp/templates/template_v3.py b/dsp/templates/template_v3.pyindex 4ae0a9c3e..fb097f877 100644--- a/dsp/templates/template_v3.py+++ b/dsp/templates/template_v3.py@@ -28,6 +28,7 @@ class Template(TemplateV2):self.fields: list[Field] = []self.format_handlers: dict[str, Callable] = {"context": passages2text,+ "passages": passages2text,"answers": format_answers,}@@ -63,3 +64,10 @@ class Template(TemplateV2):# print("here?", self.instructions == other.instructions, self.kwargs == other.kwargs)return self.instructions == other.instructions and self.kwargs == other.kwargs++ def __str__(self) -> str:+ # field names+ field_names = [field.name for field in self.fields]++ return f"Template({self.instructions}, {field_names})"+diff --git a/dsp/utils/settings.py b/dsp/utils/settings.pyindex 3d27eb384..120154fdd 100644--- a/dsp/utils/settings.py+++ b/dsp/utils/settings.py@@ -31,7 +31,8 @@ class Settings(object):force_reuse_cached_compilation=False,compiling=False,skip_logprobs=False,- trace=None+ trace=None,+ release=0,)cls._instance.__append(config)diff --git a/dspy/datasets/hotpotqa.py b/dspy/datasets/hotpotqa.pyindex e1e0256ba..5d68a0689 100644--- a/dspy/datasets/hotpotqa.py+++ b/dspy/datasets/hotpotqa.py@@ -49,6 +49,9 @@ class HotPotQA(Dataset):for raw_example in hf_official_dev:assert raw_example['level'] == 'hard'example = {k: raw_example[k] for k in ['id', 'question', 'answer', 'type', 'supporting_facts']}+ if 'supporting_facts' in example:+ example['gold_titles'] = set(example['supporting_facts']['title'])+ del example['supporting_facts']test.append(example)self._test = testdiff --git a/dspy/predict/__init__.py b/dspy/predict/__init__.pyindex bda4e41a2..7683a2f30 100644--- a/dspy/predict/__init__.py+++ b/dspy/predict/__init__.py@@ -1,3 +1,6 @@from .predict import Predictfrom .chain_of_thought import ChainOfThought-from .aggregation import majority\ No newline at end of file+from .multi_chain_comparison import MultiChainComparison+from .chain_of_thought_with_hint import ChainOfThoughtWithHint+from .react import ReAct+from .aggregation import majoritydiff --git a/dspy/predict/aggregation.py b/dspy/predict/aggregation.pyindex 4b44675e1..2212900c2 100644--- a/dspy/predict/aggregation.py+++ b/dspy/predict/aggregation.py@@ -9,9 +9,10 @@ def majority(prediction_or_completions, normalize=default_normalize, field=None)"""Returns the most common completion for the target field (or the last field) in the signature.When normalize returns None, that completion is ignored.+ In case of a tie, earlier completion are prioritized."""- assert isinstance(prediction_or_completions, Prediction) or isinstance(prediction_or_completions, Completions)+ assert any(isinstance(prediction_or_completions, t) for t in [Prediction, Completions, list])input_type = type(prediction_or_completions)# Get the completions@@ -20,8 +21,17 @@ def majority(prediction_or_completions, normalize=default_normalize, field=None)else:completions = prediction_or_completions+ try:+ signature = completions.signature+ except:+ signature = None++ try:+ field = field if field else signature.fields[-1].output_variable+ except:+ field = field if field else list(completions[0].keys())[-1]+# Normalize- field = field if field else completions.signature.fields[-1].output_variablenormalize = normalize if normalize else lambda x: xnormalized_values = [normalize(completion[field]) for completion in completions]normalized_values_ = [x for x in normalized_values if x is not None]@@ -38,8 +48,8 @@ def majority(prediction_or_completions, normalize=default_normalize, field=None)if normalize(completion[field]) == majority_value:break- if input_type == Prediction:- return Prediction.from_completions([completion], signature=completions.signature)+ # if input_type == Prediction:+ return Prediction.from_completions([completion], signature=signature)- return Completions([completion], signature=completions.signature)+ return Completions([completion], signature=signature)diff --git a/dspy/predict/chain_of_thought.py b/dspy/predict/chain_of_thought.pyindex d41003c8d..4943856b9 100644--- a/dspy/predict/chain_of_thought.py+++ b/dspy/predict/chain_of_thought.py@@ -6,6 +6,27 @@ from .predict import Predict# TODO: FIXME: Insert this right before the *first* output field. Also rewrite this to use the new signature system.+# TODO: This shouldn't inherit from Predict. It should be a module that has one or two predictors.+# Let's focus on the activated case. It's a predictor with the expanded signature.+# Now, when deactivated, it's a predictor with the original signature.+# When activate is None, though, we need the expanded one but during forward we need to pass the right signature.++"""+class ChainOfThought(dspy.Module):+ def __init__(self, signature):++ input_fields, output_fields = dspy.process_signature(signature)+ output_fields = dict(rationale=dspy.OutputField(prefix="Reasoning: Let's think step by step."), **output_fields)+ self.signature = dspy.Signature(input_fields, output_fields)++ self.predict = dspy.Predict(self.signature)++ def forward(self, **kwargs):+ return self.predict(**kwargs)++# How this should look like. But with also passing signature=simpler_signature to the predict module *if* deactivated.+"""+class ChainOfThought(Predict):def __init__(self, signature, rationale_type=None, activated=True, **config):super().__init__(signature, **config)@@ -38,4 +59,4 @@ class ChainOfThought(Predict):TODO: In principle, we can update the field's prefix during forward too to fill any thing based on the input args.IF the user didn't overwrite our default rationale_type.-"""\ No newline at end of file+"""diff --git a/dspy/predict/chain_of_thought_with_hint.py b/dspy/predict/chain_of_thought_with_hint.pynew file mode 100644index 000000000..b968d0bd9--- /dev/null+++ b/dspy/predict/chain_of_thought_with_hint.py@@ -0,0 +1,50 @@+import dsp++from .predict import Predict++++# TODO: FIXME: Insert this right before the *first* output field. Also rewrite this to use the new signature system.++class ChainOfThoughtWithHint(Predict):+ def __init__(self, signature, rationale_type=None, activated=True, **config):+ super().__init__(signature, **config)++ self.activated = activated++ signature = self.signature+ *keys, last_key = signature.kwargs.keys()++ DEFAULT_HINT_TYPE = dsp.Type(prefix="Hint:", desc="${hint}")++ DEFAULT_RATIONALE_TYPE = dsp.Type(prefix="Reasoning: Let's think step by step in order to",+ desc="${produce the " + last_key + "}. We ...")++ rationale_type = rationale_type or DEFAULT_RATIONALE_TYPE++ extended_kwargs1 = {key: signature.kwargs[key] for key in keys}+ extended_kwargs1.update({'rationale': rationale_type, last_key: signature.kwargs[last_key]})++ extended_kwargs2 = {key: signature.kwargs[key] for key in keys}+ extended_kwargs2.update({'hint': DEFAULT_HINT_TYPE, 'rationale': rationale_type, last_key: signature.kwargs[last_key]})++ self.extended_signature1 = dsp.Template(signature.instructions, **extended_kwargs1)+ self.extended_signature2 = dsp.Template(signature.instructions, **extended_kwargs2)++ def forward(self, **kwargs):+ signature = self.signature++ if self.activated is True or (self.activated is None and isinstance(dsp.settings.lm, dsp.GPT3)):+ if 'hint' in kwargs and kwargs['hint']:+ signature = self.extended_signature2+ else:+ signature = self.extended_signature1++ return super().forward(signature=signature, **kwargs)+++"""+TODO: In principle, we can update the field's prefix during forward too to fill any thing based on the input args.++IF the user didn't overwrite our default rationale_type.+"""\ No newline at end of filediff --git a/dspy/predict/multi_chain_comparison.py b/dspy/predict/multi_chain_comparison.pynew file mode 100644index 000000000..99c2b43c5--- /dev/null+++ b/dspy/predict/multi_chain_comparison.py@@ -0,0 +1,38 @@+from .predict import Predict+from ..primitives.program import Module++import dsp++class MultiChainComparison(Module):+ def __init__(self, signature, M=3, temperature=0.7, **config):+ super().__init__()++ self.M = M+ signature = Predict(signature).signature+ *keys, last_key = signature.kwargs.keys()++ extended_kwargs = {key: signature.kwargs[key] for key in keys}++ for idx in range(M):+ candidate_type = dsp.Type(prefix=f"Student Attempt #{idx+1}:", desc="${reasoning attempt}")+ extended_kwargs.update({f'reasoning_attempt_{idx+1}': candidate_type})++ rationale_type = dsp.Type(prefix="Accurate Reasoning: Thank you everyone. Let's now holistically", desc="${corrected reasoning}")+ extended_kwargs.update({'rationale': rationale_type, last_key: signature.kwargs[last_key]})++ signature = dsp.Template(signature.instructions, **extended_kwargs)+ self.predict = Predict(signature, temperature=temperature, **config)+ self.last_key = last_key++ def forward(self, completions, **kwargs):+ attempts = []++ for c in completions:+ rationale = c.rationale.strip().split('\n')[0].strip()+ answer = c[self.last_key].strip().split('\n')[0].strip()+ attempts.append(f"«I'm trying to {rationale} I'm not sure but my prediction is {answer}»")++ assert len(attempts) == self.M, len(attempts)++ kwargs = {**{f'reasoning_attempt_{idx+1}': attempt for idx, attempt in enumerate(attempts)}, **kwargs}+ return self.predict(**kwargs)diff --git a/dspy/predict/react.py b/dspy/predict/react.pynew file mode 100644index 000000000..b31e4349d--- /dev/null+++ b/dspy/predict/react.py@@ -0,0 +1,84 @@+import dsp+import dspy+from ..primitives.program import Module+from .predict import Predict++# TODO: Simplify a lot.+# TODO: Divide Action and Action Input like langchain does for ReAct.+++class ReAct(Module):+ def __init__(self, signature, max_iters=5, num_results=3, tools=None):+ super().__init__()+ self.signature = signature = dspy.Predict(signature).signature+ self.max_iters = max_iters++ self.tools = tools or [dspy.Retrieve(k=num_results)]+ self.tools = {tool.name: tool for tool in self.tools} #if isinstance(self.tools, list) else self.tools++ self.input_fields = {k: v for k, v in self.signature.kwargs.items() if isinstance(v, dspy.InputField)}+ self.output_fields = {k: v for k, v in self.signature.kwargs.items() if isinstance(v, dspy.OutputField)}++ inputs, outputs = signature.fields[:-1], signature.fields[-1:]++ inputs_ = ', '.join([f"`{field.input_variable}`" for field in inputs])+ outputs_ = ', '.join([f"`{field.output_variable}`" for field in outputs])++ assert len(outputs) == 1, "ReAct only supports one output field."++ instr = []+ instr.append(f"You will be given {inputs_} and you will respond with {outputs_}.\n")+ instr.append("To do this, you will interleave Thought, Action, and Observation steps.\n")+ instr.append("Thought can reason about the current situation, and Action can be the following types:\n")++ self.tools['Finish'] = dspy.Example(name="Finish", input_variable=outputs_.strip('`'), desc=f"returns the final {outputs_} and finishes the task")++ for idx, tool in enumerate(self.tools):+ tool = self.tools[tool]+ instr.append(f"({idx+1}) {tool.name}[{tool.input_variable}], which {tool.desc}")++ instr = '\n'.join(instr)+ self.react = [Predict(dsp.Template(instr, **self._generate_signature(i))) for i in range(1, max_iters + 1)]++ def _generate_signature(self, iters):+ signature_dict = {}+ for key, val in self.input_fields.items():+ signature_dict[key] = val++ for j in range(1, iters + 1):+ signature_dict[f"Thought_{j}"] = dspy.OutputField(prefix=f"Thought {j}:", desc="next steps to take based on last observation")++ tool_list = 'or '.join([f"{tool.name}[{tool.input_variable}]" for tool in self.tools.values() if tool.name != 'Finish'])+ signature_dict[f"Action_{j}"] = dspy.OutputField(prefix=f"Action {j}:", desc=f"always either {tool_list} or, when done, Finish[answer]")++ if j < iters:+ signature_dict[f"Observation_{j}"] = dspy.OutputField(prefix=f"Observation {j}:", desc="observations based on action", format=dsp.passages2text)++ return signature_dict++ def act(self, output, hop):+ try:+ action = output[f"Action_{hop+1}"]+ action_name, action_val = action.strip().split('\n')[0].split('[', 1)+ action_val = action_val.rsplit(']', 1)[0]++ if action_name == 'Finish': return action_val++ output[f"Observation_{hop+1}"] = self.tools[action_name](action_val).passages++ except Exception as e:+ output[f"Observation_{hop+1}"] = "Failed to parse action. Bad formatting or incorrect action name."+++ def forward(self, **kwargs):+ args = {key: kwargs[key] for key in self.input_fields.keys() if key in kwargs}++ for hop in range(self.max_iters):+ # with dspy.settings.context(show_guidelines=(i <= 2)):+ output = self.react[hop](**args)++ if action_val := self.act(output, hop): break+ args.update(output)++ # assumes only 1 output field for now - TODO: handling for multiple output fields+ return dspy.Prediction(**{list(self.output_fields.keys())[0]: action_val or ''})diff --git a/dspy/primitives/module.py b/dspy/primitives/module.pyindex 2ad125d8f..866fb2009 100644--- a/dspy/primitives/module.py+++ b/dspy/primitives/module.py@@ -26,8 +26,10 @@ class BaseModule:add_parameter(name, value)elif isinstance(value, BaseModule):- for sub_name, param in value.named_parameters():- add_parameter(f"{name}.{sub_name}", param)+ # When a sub-module is pre-compiled, keep it frozen.+ if not value._compiled:+ for sub_name, param in value.named_parameters():+ add_parameter(f"{name}.{sub_name}", param)elif isinstance(value, (list, tuple)):for idx, item in enumerate(value):diff --git a/dspy/retrieve/retrieve.py b/dspy/retrieve/retrieve.pyindex 0d62cdbba..4f61ba8ae 100644--- a/dspy/retrieve/retrieve.py+++ b/dspy/retrieve/retrieve.py@@ -6,6 +6,10 @@ from dspy.primitives.prediction import Predictionclass Retrieve(Parameter):+ name = "Search"+ input_variable = "query"+ desc = "takes a search query and returns one or more potentially relevant passages from a corpus"+def __init__(self, k=3):self.stage = random.randbytes(8).hex()self.k = k@@ -26,6 +30,11 @@ class Retrieve(Parameter):def forward(self, query_or_queries):queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries+ queries = [query.strip().split('\n')[0].strip() for query in queries]+++ # print(queries)+ # TODO: Consider removing any quote-like markers that surround the query too.passages = dsp.retrieveEnsemble(queries, k=self.k)return Prediction(passages=passages)diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.pyindex f742addb8..098acfd02 100644--- a/dspy/signatures/signature.py+++ b/dspy/signatures/signature.py@@ -48,8 +48,10 @@ class SignatureMeta(type):return f'{cls.__name__}\n' + '\n'.join(s)+class Signature(metaclass=SignatureMeta):- pass+ def __eq__(self, __value: object) -> bool:+ return self._template == __value._templatediff --git a/dspy/teleprompt/__init__.py b/dspy/teleprompt/__init__.pyindex f784f6266..b4543a8e6 100644--- a/dspy/teleprompt/__init__.py+++ b/dspy/teleprompt/__init__.py@@ -2,4 +2,5 @@ from .teleprompt import *from .bootstrap import *from .vanilla import *from .random_search import *-from .finetune import *\ No newline at end of file+from .finetune import *+diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.pyindex 9f42ff9a7..14588592d 100644--- a/dspy/teleprompt/bootstrap.py+++ b/dspy/teleprompt/bootstrap.py@@ -16,14 +16,15 @@ from dspy.evaluate.evaluate import Evaluate# NOTE: Notice the places where we don't shuffle examples. I do like that this one doesn't shuffle.# Other ones that consider options may want to use both unshuffled and then shuffle a few times, when considering candidates.-# TODO: the max_rounds should be supported here. First round is the same way it is now: minimal intervention.-# Past that first round, though, we can set the temperature to be high and we need to use branch_idx to-# get past the cache. In principle, we can also sample multiple outputs from the final generation step+# TODO: the max_rounds via branch_idx to get past the cache, not just temperature.+# In principle, we can also sample multiple outputs from the final generation step# (or even each step, in case the validation function just wants *one* thing that works, but nah)# and try them all. Having a pretty solid guess on the "final step" of each example isn't hard by the second round,# in the sense that we have the trace from the first round. (Yes it may change but that's an edge case that# won't hurt our "best effort" guarantees.)+# TODO: When this bootstraps for another teleprompter like finetune, we want all demos we gather.+# But when it's for direct use we may want to sample ONE demo per predictor--example pair. This is important for "multi-use" modules.# TODO: Add baselines=[...]@@ -64,9 +65,11 @@ class BootstrapFewShot(Teleprompter):name2predictor, predictor2name = {}, {}student, teacher = self.student, self.teacher+ assert len(student.predictors()) == len(teacher.predictors()), "Student and teacher must have the same number of predictors."+for (name1, predictor1), (name2, predictor2) in zip(student.named_predictors(), teacher.named_predictors()):assert name1 == name2, "Student and teacher must have the same program structure."- assert predictor1.signature == predictor2.signature, "Student and teacher must have the same signatures."+ assert predictor1.signature == predictor2.signature, f"Student and teacher must have the same signatures. {type(predictor1.signature)} != {type(predictor2.signature)}"assert id(predictor1) != id(predictor2), "Student and teacher must be different objects."name2predictor[name1] = None # dict(student=predictor1, teacher=predictor2)@@ -111,28 +114,29 @@ class BootstrapFewShot(Teleprompter):teacher = self.teacher #.deepcopy()predictor_cache = {}- with dsp.settings.context(trace=[], **self.teacher_settings):- lm = dsp.settings.lm- lm = lm.copy(temperature=0.7 + 0.001 * round_idx) if round_idx > 0 else lm- new_settings = dict(lm=lm) if round_idx > 0 else {}+ try:+ with dsp.settings.context(trace=[], **self.teacher_settings):+ lm = dsp.settings.lm+ lm = lm.copy(temperature=0.7 + 0.001 * round_idx) if round_idx > 0 else lm+ new_settings = dict(lm=lm) if round_idx > 0 else {}- with dsp.settings.context(**new_settings):- for name, predictor in teacher.named_predictors():- predictor_cache[name] = predictor.demos- predictor.demos = [x for x in predictor.demos if x != example]+ with dsp.settings.context(**new_settings):+ for name, predictor in teacher.named_predictors():+ predictor_cache[name] = predictor.demos+ predictor.demos = [x for x in predictor.demos if x != example]- prediction = teacher(**example.inputs())- trace = dsp.settings.trace+ prediction = teacher(**example.inputs())+ trace = dsp.settings.trace- for name, predictor in teacher.named_predictors():- predictor.demos = predictor_cache[name]+ for name, predictor in teacher.named_predictors():+ predictor.demos = predictor_cache[name]- try:- success = (self.metric is None) or self.metric(example, prediction, trace)- # print(success, example, prediction)+ success = (self.metric is None) or self.metric(example, prediction, trace)+ # print(success, example, prediction)except Exception as e:success = False- print(f'Failed to evaluate example {example.dspy_uuid} with {self.metric} due to {e}.')+ # FIXME: remove the reliance on uuid here so the error is printed+ print(f'Failed to run or to evaluate example {example} with {self.metric} due to {e}.')if success:for step in trace:@@ -147,6 +151,8 @@ class BootstrapFewShot(Teleprompter):try:predictor_name = self.predictor2name[id(predictor)]except KeyError as e:+ continue # FIXME: !+# TODO: Look closer into this. It's a bit tricky to reproduce.print(f'Failed to find predictor {predictor} in {self.predictor2name}.')print('Are you doing this in a notebook (Jupyter)? This might be caused by redefining values by rerunning cells.')@@ -168,6 +174,11 @@ class BootstrapFewShot(Teleprompter):sample_size = max(0, sample_size)raw_demos = rng.sample(raw_demos, sample_size)- predictor.demos = augmented_demos + raw_demos++ import dspy+ if dspy.settings.release >= 20230928:+ predictor.demos = raw_demos + augmented_demos+ else:+ predictor.demos = augmented_demos + raw_demosreturn self.studentdiff --git a/dspy/teleprompt/ensemble.py b/dspy/teleprompt/ensemble.pynew file mode 100644index 000000000..f52dc1021--- /dev/null+++ b/dspy/teleprompt/ensemble.py@@ -0,0 +1,40 @@+import dsp+import tqdm+import random++from dspy.teleprompt.teleprompt import Teleprompter++"""+TODO: The EnsembledProgram should actually imitate the structure of the individual programs (IF they are all compatible). This allows compiling with an ensemble program as a (singular) teacher. Basically the top majority-compatible trace will end up being used, if dspy.majority is the reduce_fn.+"""++class Ensemble(Teleprompter):+ def __init__(self, *, reduce_fn=None, size=None, deterministic=False):+ """A common reduce_fn is dspy.majority."""++ assert deterministic is False, "TODO: Implement example hashing for deterministic ensemble."++ self.reduce_fn = reduce_fn+ self.size = size+ self.deterministic = deterministic++ def compile(self, programs):+ size = self.size+ reduce_fn = self.reduce_fn++ import dspy+ class EnsembledProgram(dspy.Module):+ def __init__(self):+ super().__init__()+ self.programs = programs++ def forward(self, *args, **kwargs):+ programs = random.sample(self.programs, size) if size else self.programs+ outputs = [prog(*args, **kwargs) for prog in programs]++ if reduce_fn:+ return reduce_fn(outputs)++ return outputs++ return EnsembledProgram()diff --git a/dspy/teleprompt/finetune.py b/dspy/teleprompt/finetune.pyindex 4c7e9571b..a84655bf7 100644--- a/dspy/teleprompt/finetune.py+++ b/dspy/teleprompt/finetune.py@@ -54,37 +54,40 @@ class BootstrapFinetune(Teleprompter):metric = metric or (lambda *args: True)self.teleprompter = BootstrapFewShot(metric=metric,max_bootstrapped_demos=999999,- max_labeled_demos=0, # FIXME: TODO: Make this zero? or param, 16 by default?+ max_labeled_demos=0, # FIXME: TODO: Make this zero? or param, with default as 16 or 0?teacher_settings=teacher_settings)def compile(self, student, *, teacher=None, trainset, valset=None,- target='t5-large', bsize=12, accumsteps=1, lr=5e-5, epochs=1, bf16=False):+ target='t5-large', bsize=12, accumsteps=1, lr=5e-5, epochs=1, bf16=False, int8=False, peft=False):# It's usually better to supply a few-shot teacher, rather than uncompiled module (the student).if teacher is None:print("WARNING: Using a vanilla teacher. ""Are you sure you want to use BootstrapFinetune without a compiled teacher?")- # Dummy compilation to get bootstraps.- compiled = self.teleprompter.compile(student, teacher=teacher, trainset=trainset)- multitask = self.multitask- #- # Prepare finetune <prompt, completion> pairs.- #+ teachers = teacher if isinstance(teacher, list) else [teacher]finetune_data = {}- for name, predictor in compiled.named_predictors():- name_ = 'all' if multitask else name- finetune_data[name_] = [] if name_ not in finetune_data else finetune_data[name_]+ for teacher in teachers:+ # Dummy compilation to get bootstraps.+ compiled = self.teleprompter.compile(student, teacher=teacher, trainset=trainset)+ multitask = self.multitask++ # Prepare finetune <prompt, completion> pairs.+ for name, predictor in compiled.named_predictors():+ name_ = 'all' if multitask else name+ finetune_data[name_] = [] if name_ not in finetune_data else finetune_data[name_]- for demo in predictor.demos:- demo = dict(demo)- completion = demo.pop(predictor.signature.fields[-1].output_variable)- prompt = predictor.signature.query(dsp.Example(demos=[], **demo)).strip()+ for demo in predictor.demos:+ demo = dict(demo)- finetune_data[name_].append(dict(prompt=prompt, completion=completion))+ # TODO: FIXME: generalize.+ completion = demo.pop(predictor.signature.fields[-1].output_variable)+ prompt = predictor.signature.query(dsp.Example(demos=[], **demo)).strip()++ finetune_data[name_].append(dict(prompt=prompt, completion=completion))for name_ in finetune_data:random.Random(0).shuffle(finetune_data[name_])@@ -115,9 +118,10 @@ class BootstrapFinetune(Teleprompter):import stringcompiler_config = {'save': ''.join(random.Random(time.time()).choices(string.ascii_uppercase + string.digits, k=13)), # https://stackoverflow.com/a/2257449/1493011- 'peft': False,+ 'peft': peft,'fp16': False,'bf16': bf16,+ 'int8': int8,'fid': False,'rationale': False,'batch_size': bsize,@@ -135,9 +139,11 @@ class BootstrapFinetune(Teleprompter):training_data_path = finetune_paths[name]compiler_config_ = dict(compiler_config)compiler_config_['save'] = compiler_config['save'] + '.' + name- best_ckpt_path = finetune_hf(training_data_path, target, compiler_config_)+ best_ckpt_path, tokenized_data = finetune_hf(training_data_path, target, compiler_config_)++ print(f"#> Best checkpoint path: {best_ckpt_path} for {name}")finetune_models[name] = dsp.HFModel(model=target, checkpoint=best_ckpt_path) # best_ckpt_path-+## Set the LMs to the finetuned ones, per module#@@ -151,8 +157,9 @@ class BootstrapFinetune(Teleprompter):# TODO: FIXME: When we assign .lm, the Predict.forward will also set only_query=True.# This is correct for here but we may want to make it more explicitly restricted to finetuned models.+ print(f"Assigning the LM of predictor {name}.")+predictor2.lm = finetune_models[name]assert predictor2.demos == []return compiled2-diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.pyindex c2baf3785..f027f83fe 100644--- a/dspy/teleprompt/random_search.py+++ b/dspy/teleprompt/random_search.py@@ -48,7 +48,7 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):# print("Going to sample", self.max_num_traces, "traces in total.")print("Will attempt to train", self.num_candidate_sets, "candidate sets.")- def compile(self, student, *, teacher=None, trainset, valset=None):+ def compile(self, student, *, teacher=None, trainset, valset=None, restrict=None):self.trainset = trainsetself.valset = valset or trainset # TODO: FIXME: Note this choice.@@ -57,6 +57,10 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):score_data = []for seed in range(-3, self.num_candidate_sets):+ if (restrict is not None) and (seed not in restrict):+ print(seed, restrict)+ continue+trainset2 = list(self.trainset)if seed == -3:@@ -96,32 +100,38 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):all_subscores.append(subscores)print('Score:', score, 'for set:', [len(predictor.demos) for predictor in program2.predictors()])- scores.append(score)-- print(f"Scores so far: {scores}")- if score >= max(scores):+ if len(scores) == 0 or score > max(scores):print('New best score:', score, 'for seed', seed)best_program = program2+ scores.append(score)+ print(f"Scores so far: {scores}")+print('Best score:', max(scores))- score_data.append((score, subscores))+ score_data.append((score, subscores, seed, program2))if len(score_data) > 2: # We check if there are at least 3 scores to considerfor k in [1, 2, 3, 5, 8, 9999]:top_3_scores = sorted(score_data, key=lambda x: x[0], reverse=True)[:k]# Transpose the subscores to get max per entry and then calculate their average- transposed_subscores = zip(*[subscores for _, subscores in top_3_scores if subscores])+ transposed_subscores = zip(*[subscores for _, subscores, *_ in top_3_scores if subscores])avg_of_max_per_entry = sum(max(entry) for entry in transposed_subscores) / len(top_3_scores[0][1])print(f'Average of max per entry across top {k} scores: {avg_of_max_per_entry}')-+if self.stop_at_score is not None and score >= self.stop_at_score:print(f"Stopping early because score {score} is >= stop_at_score {self.stop_at_score}")break+ # To best program, attach all program candidates in decreasing average score+ best_program.candidate_programs = score_data+ best_program.candidate_programs = sorted(best_program.candidate_programs, key=lambda x: x[0], reverse=True)++ print(len(best_program.candidate_programs), "candidate programs found.")+return best_programdiff --git a/dspy/teleprompt/vanilla.py b/dspy/teleprompt/vanilla.pyindex 619733691..068343fa0 100644--- a/dspy/teleprompt/vanilla.py+++ b/dspy/teleprompt/vanilla.py@@ -8,7 +8,7 @@ class LabeledFewShot(Teleprompter):def __init__(self, k=16):self.k = k- def compile(self, student, *, trainset):+ def compile(self, student, *, trainset, sample=True):self.student = student.reset_copy()self.trainset = trainset@@ -18,7 +18,10 @@ class LabeledFewShot(Teleprompter):rng = random.Random(0)for predictor in self.student.predictors():- predictor.demos = rng.sample(self.trainset, min(self.k, len(self.trainset)))+ if sample:+ predictor.demos = rng.sample(self.trainset, min(self.k, len(self.trainset)))+ else:+ predictor.demos = self.trainset[:min(self.k, len(self.trainset))]return self.student