DSPy: The framework for programming—not prompting—language models
Python
Removing dead code (WIP)
Omar Khattab7484d1ebc1e0
diff --git a/dsp/modules/lm.py b/dsp/modules/lm.pyindex 7f7966d4b..eb5ebb54e 100644--- a/dsp/modules/lm.py+++ b/dsp/modules/lm.py@@ -26,7 +26,7 @@ class LM(ABC):return self.basic_request(prompt, **kwargs)def print_green(self, text: str, end: str = "\n"):- return "\x1b[32m" + str(text) + "\x1b[0m" + end+ return "\n" + "\x1b[32m" + str(text) + "\x1b[0m" + enddef print_red(self, text: str, end: str = "\n"):return "\x1b[31m" + str(text) + "\x1b[0m" + end@@ -113,7 +113,7 @@ class LM(ABC):text = choices[0]else:text = choices[0]["text"]- printing_value += self.print_green(text, end="")+ printing_value += self.print_green(text.lstrip(), end="")if len(choices) > 1 and isinstance(choices, list):printing_value += self.print_red(@@ -134,4 +134,4 @@ class LM(ABC):kwargs = {**self.kwargs, **kwargs}model = kwargs.pop("model")- return self.__class__(model=model, **kwargs)+ return self.__class__(model=model, **kwargs)\ No newline at end of filediff --git a/dsp/primitives/compiler.py b/dsp/primitives/compiler.pyindex 625ae23ac..a004a9834 100644--- a/dsp/primitives/compiler.py+++ b/dsp/primitives/compiler.py@@ -1,173 +1,173 @@-import os-import random-import subprocess-import time+# import os+# import random+# import subprocess+# import time-import tqdm-import ujson-from datasets.fingerprint import Hasher+# import tqdm+# import ujson+# from datasets.fingerprint import Hasher-import dsp+# import dsp-if os.environ.get('DSP_NOTEBOOK_CACHEDIR'):- training_data_directory = os.path.join(os.environ.get('DSP_NOTEBOOK_CACHEDIR'), 'compiler')-else:- training_data_directory = 'cache/compiler'+# if os.environ.get('DSP_NOTEBOOK_CACHEDIR'):+# training_data_directory = os.path.join(os.environ.get('DSP_NOTEBOOK_CACHEDIR'), 'compiler')+# else:+# training_data_directory = 'cache/compiler'-compilations_assumed_to_exist={'ft-zvEdzQVQ5xwlxvNPrxl6kpnw': 'ada:ft-stanfordpraglab-2023-02-09-19-50-49'}+# compilations_assumed_to_exist={'ft-zvEdzQVQ5xwlxvNPrxl6kpnw': 'ada:ft-stanfordpraglab-2023-02-09-19-50-49'}-def openai_check_finetune(jobname):- if dsp.settings.force_reuse_cached_compilation and jobname in compilations_assumed_to_exist:- return compilations_assumed_to_exist[jobname]+# def openai_check_finetune(jobname):+# if dsp.settings.force_reuse_cached_compilation and jobname in compilations_assumed_to_exist:+# return compilations_assumed_to_exist[jobname]- command = f"""openai api fine_tunes.get -i {jobname}"""- print(command)+# command = f"""openai api fine_tunes.get -i {jobname}"""+# print(command)- result = subprocess.run(command.split(), stdout=subprocess.PIPE, check=False)- output = result.stdout.decode("utf-8").strip()+# result = subprocess.run(command.split(), stdout=subprocess.PIPE, check=False)+# output = result.stdout.decode("utf-8").strip()- try:- output = ujson.loads(output)- if output['status'] == 'succeeded':- return output['fine_tuned_model']+# try:+# output = ujson.loads(output)+# if output['status'] == 'succeeded':+# return output['fine_tuned_model']- if output['status'] in ['pending', 'running']:- print(f'Compiling, run ```openai api fine_tunes.follow -i {jobname}``` for details...')- time.sleep(60)- return openai_check_finetune(jobname)- except:- pass+# if output['status'] in ['pending', 'running']:+# print(f'Compiling, run ```openai api fine_tunes.follow -i {jobname}``` for details...')+# time.sleep(60)+# return openai_check_finetune(jobname)+# except:+# pass- return False+# return False-def convert_to_training_point2(y, inputs, outputs, template):- assert len(inputs) + len(outputs) == len(template.fields)+# def convert_to_training_point2(y, inputs, outputs, template):+# assert len(inputs) + len(outputs) == len(template.fields)- y_ = dsp.Example(**{f: y[f] for f in inputs}, demos=[])- prompt = template(y_, show_guidelines=False)+# y_ = dsp.Example(**{f: y[f] for f in inputs}, demos=[])+# prompt = template(y_, show_guidelines=False)- completion = y[outputs[0]]- output_fields = template.fields[len(inputs):]+# completion = y[outputs[0]]+# output_fields = template.fields[len(inputs):]- for field in output_fields[1:]:- completion += f"\n\n{field.name} " + y[field.output_variable]+# for field in output_fields[1:]:+# completion += f"\n\n{field.name} " + y[field.output_variable]- completion = " " + completion + " </s>"- return {'prompt': prompt, 'completion': completion}+# completion = " " + completion + " </s>"+# return {'prompt': prompt, 'completion': completion}-def simulate(program, input_examples):- training_data = []+# def simulate(program, input_examples):+# training_data = []- for input_example in tqdm.tqdm(input_examples):- prediction = program(input_example)+# for input_example in tqdm.tqdm(input_examples):+# prediction = program(input_example)- if prediction is not None:- # assert len(prediction.compiling_stages) == 2, "TMP"- for stage in prediction.compiling_stages:- name, template, inputs, outputs = stage['name'], stage['template'], stage['inputs'], stage['outputs']- training_data.append(convert_to_training_point2(prediction.get(name), inputs, outputs, template))+# if prediction is not None:+# # assert len(prediction.compiling_stages) == 2, "TMP"+# for stage in prediction.compiling_stages:+# name, template, inputs, outputs = stage['name'], stage['template'], stage['inputs'], stage['outputs']+# training_data.append(convert_to_training_point2(prediction.get(name), inputs, outputs, template))- r = random.Random(0)- r.shuffle(training_data)+# r = random.Random(0)+# r.shuffle(training_data)- return training_data+# return training_data-def openai_finetune_(name, target):- training_data_path = name_to_path(name)+# def openai_finetune_(name, target):+# training_data_path = name_to_path(name)- # Launch the fine-tune on the path- command = f"""openai api fine_tunes.create -t {training_data_path} -m {target} --n_epochs 4 --learning_rate_multiplier 0.05 --no_check_if_files_exist"""- print(command)+# # Launch the fine-tune on the path+# command = f"""openai api fine_tunes.create -t {training_data_path} -m {target} --n_epochs 4 --learning_rate_multiplier 0.05 --no_check_if_files_exist"""+# print(command)- # command = """python script.py"""- process = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)+# # command = """python script.py"""+# process = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)- while line := process.stdout.readline().decode().strip():- if 'created fine-tune:' in line.lower():- jobname = line.split()[-1]- break+# while line := process.stdout.readline().decode().strip():+# if 'created fine-tune:' in line.lower():+# jobname = line.split()[-1]+# break- # if 'costs $' in line.lower():- # cost = line.split()[-1]- # break+# # if 'costs $' in line.lower():+# # cost = line.split()[-1]+# # break- # assert cost[0] == '$'+# # assert cost[0] == '$'- # if float(cost[1:]) > 300:- # print(f'Got cost {cost} -- you may wanna cancel the job: openai api fine_tunes.cancel -i {jobname}')+# # if float(cost[1:]) > 300:+# # print(f'Got cost {cost} -- you may wanna cancel the job: openai api fine_tunes.cancel -i {jobname}')- # print(cost)+# # print(cost)- print(jobname)+# print(jobname)- # Block until it's done- ft = openai_check_finetune(jobname)- assert ft, ft+# # Block until it's done+# ft = openai_check_finetune(jobname)+# assert ft, ft- # Return its name- return (jobname, ft)+# # Return its name+# return (jobname, ft)-def openai_finetune(name, target):- print(name)- training_data_path = name_to_path(name)- training_data_path += '.model'+# def openai_finetune(name, target):+# print(name)+# training_data_path = name_to_path(name)+# training_data_path += '.model'- # if path + stuff exists, load the tuple from it- try:- with open(training_data_path) as f:- jobname, ft = ujson.loads(f.readline())+# # if path + stuff exists, load the tuple from it+# try:+# with open(training_data_path) as f:+# jobname, ft = ujson.loads(f.readline())- if openai_check_finetune(jobname):- return jobname, ft- except:- pass+# if openai_check_finetune(jobname):+# return jobname, ft+# except:+# pass- jobname, ft = openai_finetune_(name, target)+# jobname, ft = openai_finetune_(name, target)- with open(training_data_path, 'w') as f:- f.write(ujson.dumps((jobname, ft)) + '\n')+# with open(training_data_path, 'w') as f:+# f.write(ujson.dumps((jobname, ft)) + '\n')- return jobname, ft+# return jobname, ft-def name_to_path(name):- if not os.path.exists(training_data_directory):- os.makedirs(training_data_directory)+# def name_to_path(name):+# if not os.path.exists(training_data_directory):+# os.makedirs(training_data_directory)- training_data_path = os.path.join(training_data_directory, f'{name}.jsonl')- return training_data_path+# training_data_path = os.path.join(training_data_directory, f'{name}.jsonl')+# return training_data_path-# 3. Check that the output file name has status "success" (not deleted or non-existent). Otherwise, re-call with n = n+1.-def finetune(training_data, target):- name = Hasher.hash(training_data)- training_data_path = name_to_path(name)+# # 3. Check that the output file name has status "success" (not deleted or non-existent). Otherwise, re-call with n = n+1.+# def finetune(training_data, target):+# name = Hasher.hash(training_data)+# training_data_path = name_to_path(name)- with open(training_data_path, 'w') as f:- for line in training_data:- f.write(ujson.dumps(line) + '\n')+# with open(training_data_path, 'w') as f:+# for line in training_data:+# f.write(ujson.dumps(line) + '\n')- jobname, ft = openai_finetune(name, target)- print(ft)+# jobname, ft = openai_finetune(name, target)+# print(ft)- ft = dsp.GPT3(model=ft, stop=" </s>")- return ft+# ft = dsp.GPT3(model=ft, stop=" </s>")+# return ft-# 4. Return updated program.-def compile(program, examples, target='ada'):- training_data = simulate(program, examples)- compiled_lm = finetune(training_data, target=target)+# # 4. Return updated program.+# def compile(program, examples, target='ada'):+# training_data = simulate(program, examples)+# compiled_lm = finetune(training_data, target=target)- def compiled_program(*args, **kwargs):- with dsp.settings.context(compiled_lm=compiled_lm, compiling=False):- return program(*args, **kwargs)+# def compiled_program(*args, **kwargs):+# with dsp.settings.context(compiled_lm=compiled_lm, compiling=False):+# return program(*args, **kwargs)- compiled_program.lm = compiled_lm- return compiled_program+# compiled_program.lm = compiled_lm+# return compiled_programdiff --git a/dsp/primitives/demonstrate.py b/dsp/primitives/demonstrate.pyindex 215afc23d..33ba97932 100644--- a/dsp/primitives/demonstrate.py+++ b/dsp/primitives/demonstrate.py@@ -45,62 +45,62 @@ class Example(dotdict):return self.copy(demos=demos)-def annotate(*transformations):- """Returns an Augment function that applies the provided transformations to the Examples"""+# def annotate(*transformations):+# """Returns an Augment function that applies the provided transformations to the Examples"""- def do_augment(train, k=None, return_all=False):- rdemos = []- ademos = []+# def do_augment(train, k=None, return_all=False):+# rdemos = []+# ademos = []- for example in train: # tqdm.tqdm- raw_example = dsp.Example(example)+# for example in train: # tqdm.tqdm+# raw_example = dsp.Example(example)- if (k is not None) and len(ademos) >= k:- example = None+# if (k is not None) and len(ademos) >= k:+# example = None- for f in transformations:- if example is None:- break+# for f in transformations:+# if example is None:+# break- example = f(example)+# example = f(example)- if example is not None:- example.augmented = True- ademos.append(example)- else:- raw_example.augmented = False- rdemos.append(raw_example)+# if example is not None:+# example.augmented = True+# ademos.append(example)+# else:+# raw_example.augmented = False+# rdemos.append(raw_example)- if return_all:- return ademos + rdemos+# if return_all:+# return ademos + rdemos- return ademos+# return ademos- return do_augment+# return do_augment-def sample(train: list[Example], k: int):- """Sample k examples from train."""- rng = random.Random(dsp.settings.branch_idx)- shuffled_train = [dsp.Example(example) for example in train]- rng.shuffle(shuffled_train)+# def sample(train: list[Example], k: int):+# """Sample k examples from train."""+# rng = random.Random(dsp.settings.branch_idx)+# shuffled_train = [dsp.Example(example) for example in train]+# rng.shuffle(shuffled_train)- return shuffled_train[:k]+# return shuffled_train[:k]-def all_but(train: list[Example], x: Example) -> list[Example]:- """Removes the example x from the train set by comparing the question and history."""+# def all_but(train: list[Example], x: Example) -> list[Example]:+# """Removes the example x from the train set by comparing the question and history."""- output = [- y- for y in train- if not set.intersection(- set(x.get("history", []) + [x.question]),- set(y.get("history", []) + [y.question]),- )- ]+# output = [+# y+# for y in train+# if not set.intersection(+# set(x.get("history", []) + [x.question]),+# set(y.get("history", []) + [y.question]),+# )+# ]- return output+# return outputdef passage_match(passages: list[str], answers: list[str]) -> bool:diff --git a/dsp/primitives/inspect.py b/dsp/primitives/inspect.pyindex ebcd3b787..edc8345d5 100644--- a/dsp/primitives/inspect.py+++ b/dsp/primitives/inspect.py@@ -1,91 +1,91 @@-import inspect-import json-import random-import string--import requests---class FuncInspector:- def __init__(self):- self.calls = []--- def inspect_inner(self, func, function_calls):- def wrapper(*args, **kwargs):- result = func(*args, **kwargs)- self.merge_result(result, function_calls)- return result- return wrapper--- def inspect_func(self, func):- def wrapper(*args, **kwargs):- result = func(*args, **kwargs)- stack = inspect.stack()- function_calls = []- for i in range(len(stack)):- if stack[i][3] == "<module>":- break- if stack[i][3] != "wrapper":- function_calls.append(stack[i][3])- function_calls.reverse()- result = self.inspect_inner(result, function_calls)- return result- return wrapper+# import inspect+# import json+# import random+# import string++# import requests+++# class FuncInspector:+# def __init__(self):+# self.calls = []+++# def inspect_inner(self, func, function_calls):+# def wrapper(*args, **kwargs):+# result = func(*args, **kwargs)+# self.merge_result(result, function_calls)+# return result+# return wrapper+++# def inspect_func(self, func):+# def wrapper(*args, **kwargs):+# result = func(*args, **kwargs)+# stack = inspect.stack()+# function_calls = []+# for i in range(len(stack)):+# if stack[i][3] == "<module>":+# break+# if stack[i][3] != "wrapper":+# function_calls.append(stack[i][3])+# function_calls.reverse()+# result = self.inspect_inner(result, function_calls)+# return result+# return wrapper- def parse(self, obj, delete_empty=False):- if isinstance(obj, list):- for elem in obj:- self.parse(elem, delete_empty)- if isinstance(obj, dict):- to_delete = []- for key in obj:- if delete_empty and not obj[key] or key == "completions":- to_delete.append(key)- else:- self.parse(obj[key], delete_empty)- for key in to_delete:- obj.pop(key)--- def merge_result(self, result, function_calls):- prev_list = self.calls- prev_call = {} if not prev_list else prev_list[-1]- for call in function_calls[:-1]:- if call not in prev_call:- prev_call = {call: []}- prev_list.append(prev_call)- prev_list = prev_call[call]- prev_call = {} if not prev_list else prev_list[-1]-- example_obj = result[0]- self.parse(example_obj)- prev_list.append({ function_calls[-1]: example_obj })--- def view_data(self):- chars = string.digits + string.ascii_lowercase- id = ''.join(random.choices(chars, k=8))-- post_url = 'http://127.0.0.1:5000/log-item'- parsed_calls = self.calls.copy()- self.parse(parsed_calls, delete_empty=True)- data = {'id': id, 'content': parsed_calls}- response = requests.post(post_url, json=data)+# def parse(self, obj, delete_empty=False):+# if isinstance(obj, list):+# for elem in obj:+# self.parse(elem, delete_empty)+# if isinstance(obj, dict):+# to_delete = []+# for key in obj:+# if delete_empty and not obj[key] or key == "completions":+# to_delete.append(key)+# else:+# self.parse(obj[key], delete_empty)+# for key in to_delete:+# obj.pop(key)+++# def merge_result(self, result, function_calls):+# prev_list = self.calls+# prev_call = {} if not prev_list else prev_list[-1]+# for call in function_calls[:-1]:+# if call not in prev_call:+# prev_call = {call: []}+# prev_list.append(prev_call)+# prev_list = prev_call[call]+# prev_call = {} if not prev_list else prev_list[-1]++# example_obj = result[0]+# self.parse(example_obj)+# prev_list.append({ function_calls[-1]: example_obj })+++# def view_data(self):+# chars = string.digits + string.ascii_lowercase+# id = ''.join(random.choices(chars, k=8))++# post_url = 'http://127.0.0.1:5000/log-item'+# parsed_calls = self.calls.copy()+# self.parse(parsed_calls, delete_empty=True)+# data = {'id': id, 'content': parsed_calls}+# response = requests.post(post_url, json=data)- if response.status_code == 201:- print('Data created successfully')- else:- print(f'Error sending data to server: {response.status_code}')- return+# if response.status_code == 201:+# print('Data created successfully')+# else:+# print(f'Error sending data to server: {response.status_code}')+# return- frontend_url = f"http://localhost:3000?id={id}"- print(f"View the data here, {frontend_url}")+# frontend_url = f"http://localhost:3000?id={id}"+# print(f"View the data here, {frontend_url}")- def output_json(self, out_path):- f = open(out_path, "w")- json_object = json.dumps(self.calls, indent=2)- f.write(json_object)+# def output_json(self, out_path):+# f = open(out_path, "w")+# json_object = json.dumps(self.calls, indent=2)+# f.write(json_object)diff --git a/dsp/primitives/primitives.py b/dsp/primitives/primitives.pyindex 2e4c4bfc0..50aea522b 100644--- a/dsp/primitives/primitives.py+++ b/dsp/primitives/primitives.py@@ -1,46 +1,46 @@-from functools import wraps+# from functools import wraps-import dsp+# import dsp-# applied right to left (innermost first, like function calls)-def compose_decorators(*decorators):- def decorator(func):- for decorator in decorators[::-1]:- func = decorator(func)- return func- return decorator+# # applied right to left (innermost first, like function calls)+# def compose_decorators(*decorators):+# def decorator(func):+# for decorator in decorators[::-1]:+# func = decorator(func)+# return func+# return decorator-def shallow_copy_example_args(func):- @wraps(func)- def wrapper(*args, **kwargs):- args = [dsp.Example(arg) if isinstance(arg, dsp.Example) else arg for arg in args]- kwargs = {key: dsp.Example(value) if isinstance(value, dsp.Example) else value for key, value in kwargs.items()}- return func(*args, **kwargs)- return wrapper+# def shallow_copy_example_args(func):+# @wraps(func)+# def wrapper(*args, **kwargs):+# args = [dsp.Example(arg) if isinstance(arg, dsp.Example) else arg for arg in args]+# kwargs = {key: dsp.Example(value) if isinstance(value, dsp.Example) else value for key, value in kwargs.items()}+# return func(*args, **kwargs)+# return wrapper-transformation = shallow_copy_example_args-# transformation = compose_decorators(handle_compilation, shallow_copy_example_args)+# transformation = shallow_copy_example_args+# # transformation = compose_decorators(handle_compilation, shallow_copy_example_args)-def compiled(func):- def wrapper(*args, **kwargs):- is_to_be_compiled = True #decorator_kwargs.get('compile', False)- compiled_lm = dsp.settings.compiled_lm+# def compiled(func):+# def wrapper(*args, **kwargs):+# is_to_be_compiled = True #decorator_kwargs.get('compile', False)+# compiled_lm = dsp.settings.compiled_lm- if is_to_be_compiled and compiled_lm:- assert len(args) == 1, len(args)- example = args[0]+# if is_to_be_compiled and compiled_lm:+# assert len(args) == 1, len(args)+# example = args[0]- with dsp.settings.context(lm=compiled_lm, show_guidelines=False):- old_demos = list(example.demos)- example = func(example.copy(demos=[]), **kwargs)- return example.copy(demos=old_demos)+# with dsp.settings.context(lm=compiled_lm, show_guidelines=False):+# old_demos = list(example.demos)+# example = func(example.copy(demos=[]), **kwargs)+# return example.copy(demos=old_demos)- with dsp.settings.context(compiling=True):- return func(*args, **kwargs)+# with dsp.settings.context(compiling=True):+# return func(*args, **kwargs)- return wrapper+# return wrapperdiff --git a/dsp/templates/utils.py b/dsp/templates/utils.pyindex 53624ad2d..4bc15c058 100644--- a/dsp/templates/utils.py+++ b/dsp/templates/utils.py@@ -17,29 +17,29 @@ def passages2text(passages: Union[str, list, tuple]) -> str:return "\n".join([f"[{idx+1}] «{txt}»" for idx, txt in enumerate(passages)])-def passages2textV2(passages: Union[str, list, tuple]) -> str:- """Formats the given one or more passages into a single structured string."""- if isinstance(passages, str):- return passages-- assert type(passages) in [list, tuple]-- def psg2text(psg):- try:- title, snippet = psg.split("|", 1)- return f"Title: {title.strip()} | Snippet: «{snippet.strip()}»"- except Exception:- pass+# def passages2textV2(passages: Union[str, list, tuple]) -> str:+# """Formats the given one or more passages into a single structured string."""+# if isinstance(passages, str):+# return passages++# assert type(passages) in [list, tuple]++# def psg2text(psg):+# try:+# title, snippet = psg.split("|", 1)+# return f"Title: {title.strip()} | Snippet: «{snippet.strip()}»"+# except Exception:+# pass- return f"«{psg}»"+# return f"«{psg}»"- if len(passages) == 0:- return "N/A"+# if len(passages) == 0:+# return "N/A"- if len(passages) == 1:- return psg2text(passages[0])+# if len(passages) == 1:+# return psg2text(passages[0])- return "\n".join([f"[{idx+1}] {psg2text(txt)}" for idx, txt in enumerate(passages)])+# return "\n".join([f"[{idx+1}] {psg2text(txt)}" for idx, txt in enumerate(passages)])def format_answers(answers: Union[str, list]) -> Optional[str]: