Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Removing dead code (WIP)

Omar Khattab7484d1ebc1e0
diff --git a/dsp/modules/lm.py b/dsp/modules/lm.py
index 7f7966d4b..eb5ebb54e 100644
--- a/dsp/modules/lm.py
+++ b/dsp/modules/lm.py
@@ -26,7 +26,7 @@ class LM(ABC):
return self.basic_request(prompt, **kwargs)
def print_green(self, text: str, end: str = "\n"):
- return "\x1b[32m" + str(text) + "\x1b[0m" + end
+ return "\n" + "\x1b[32m" + str(text) + "\x1b[0m" + end
def print_red(self, text: str, end: str = "\n"):
return "\x1b[31m" + str(text) + "\x1b[0m" + end
@@ -113,7 +113,7 @@ class LM(ABC):
text = choices[0]
else:
text = choices[0]["text"]
- printing_value += self.print_green(text, end="")
+ printing_value += self.print_green(text.lstrip(), end="")
if len(choices) > 1 and isinstance(choices, list):
printing_value += self.print_red(
@@ -134,4 +134,4 @@ class LM(ABC):
kwargs = {**self.kwargs, **kwargs}
model = kwargs.pop("model")
- return self.__class__(model=model, **kwargs)
+ return self.__class__(model=model, **kwargs)
\ No newline at end of file
diff --git a/dsp/primitives/compiler.py b/dsp/primitives/compiler.py
index 625ae23ac..a004a9834 100644
--- a/dsp/primitives/compiler.py
+++ b/dsp/primitives/compiler.py
@@ -1,173 +1,173 @@
-import os
-import random
-import subprocess
-import time
+# import os
+# import random
+# import subprocess
+# import time
-import tqdm
-import ujson
-from datasets.fingerprint import Hasher
+# import tqdm
+# import ujson
+# from datasets.fingerprint import Hasher
-import dsp
+# import dsp
-if os.environ.get('DSP_NOTEBOOK_CACHEDIR'):
- training_data_directory = os.path.join(os.environ.get('DSP_NOTEBOOK_CACHEDIR'), 'compiler')
-else:
- training_data_directory = 'cache/compiler'
+# if os.environ.get('DSP_NOTEBOOK_CACHEDIR'):
+# training_data_directory = os.path.join(os.environ.get('DSP_NOTEBOOK_CACHEDIR'), 'compiler')
+# else:
+# training_data_directory = 'cache/compiler'
-compilations_assumed_to_exist={'ft-zvEdzQVQ5xwlxvNPrxl6kpnw': 'ada:ft-stanfordpraglab-2023-02-09-19-50-49'}
+# compilations_assumed_to_exist={'ft-zvEdzQVQ5xwlxvNPrxl6kpnw': 'ada:ft-stanfordpraglab-2023-02-09-19-50-49'}
-def openai_check_finetune(jobname):
- if dsp.settings.force_reuse_cached_compilation and jobname in compilations_assumed_to_exist:
- return compilations_assumed_to_exist[jobname]
+# def openai_check_finetune(jobname):
+# if dsp.settings.force_reuse_cached_compilation and jobname in compilations_assumed_to_exist:
+# return compilations_assumed_to_exist[jobname]
- command = f"""openai api fine_tunes.get -i {jobname}"""
- print(command)
+# command = f"""openai api fine_tunes.get -i {jobname}"""
+# print(command)
- result = subprocess.run(command.split(), stdout=subprocess.PIPE, check=False)
- output = result.stdout.decode("utf-8").strip()
+# result = subprocess.run(command.split(), stdout=subprocess.PIPE, check=False)
+# output = result.stdout.decode("utf-8").strip()
- try:
- output = ujson.loads(output)
- if output['status'] == 'succeeded':
- return output['fine_tuned_model']
+# try:
+# output = ujson.loads(output)
+# if output['status'] == 'succeeded':
+# return output['fine_tuned_model']
- if output['status'] in ['pending', 'running']:
- print(f'Compiling, run ```openai api fine_tunes.follow -i {jobname}``` for details...')
- time.sleep(60)
- return openai_check_finetune(jobname)
- except:
- pass
+# if output['status'] in ['pending', 'running']:
+# print(f'Compiling, run ```openai api fine_tunes.follow -i {jobname}``` for details...')
+# time.sleep(60)
+# return openai_check_finetune(jobname)
+# except:
+# pass
- return False
+# return False
-def convert_to_training_point2(y, inputs, outputs, template):
- assert len(inputs) + len(outputs) == len(template.fields)
+# def convert_to_training_point2(y, inputs, outputs, template):
+# assert len(inputs) + len(outputs) == len(template.fields)
- y_ = dsp.Example(**{f: y[f] for f in inputs}, demos=[])
- prompt = template(y_, show_guidelines=False)
+# y_ = dsp.Example(**{f: y[f] for f in inputs}, demos=[])
+# prompt = template(y_, show_guidelines=False)
- completion = y[outputs[0]]
- output_fields = template.fields[len(inputs):]
+# completion = y[outputs[0]]
+# output_fields = template.fields[len(inputs):]
- for field in output_fields[1:]:
- completion += f"\n\n{field.name} " + y[field.output_variable]
+# for field in output_fields[1:]:
+# completion += f"\n\n{field.name} " + y[field.output_variable]
- completion = " " + completion + " </s>"
- return {'prompt': prompt, 'completion': completion}
+# completion = " " + completion + " </s>"
+# return {'prompt': prompt, 'completion': completion}
-def simulate(program, input_examples):
- training_data = []
+# def simulate(program, input_examples):
+# training_data = []
- for input_example in tqdm.tqdm(input_examples):
- prediction = program(input_example)
+# for input_example in tqdm.tqdm(input_examples):
+# prediction = program(input_example)
- if prediction is not None:
- # assert len(prediction.compiling_stages) == 2, "TMP"
- for stage in prediction.compiling_stages:
- name, template, inputs, outputs = stage['name'], stage['template'], stage['inputs'], stage['outputs']
- training_data.append(convert_to_training_point2(prediction.get(name), inputs, outputs, template))
+# if prediction is not None:
+# # assert len(prediction.compiling_stages) == 2, "TMP"
+# for stage in prediction.compiling_stages:
+# name, template, inputs, outputs = stage['name'], stage['template'], stage['inputs'], stage['outputs']
+# training_data.append(convert_to_training_point2(prediction.get(name), inputs, outputs, template))
- r = random.Random(0)
- r.shuffle(training_data)
+# r = random.Random(0)
+# r.shuffle(training_data)
- return training_data
+# return training_data
-def openai_finetune_(name, target):
- training_data_path = name_to_path(name)
+# def openai_finetune_(name, target):
+# training_data_path = name_to_path(name)
- # Launch the fine-tune on the path
- command = f"""openai api fine_tunes.create -t {training_data_path} -m {target} --n_epochs 4 --learning_rate_multiplier 0.05 --no_check_if_files_exist"""
- print(command)
+# # Launch the fine-tune on the path
+# command = f"""openai api fine_tunes.create -t {training_data_path} -m {target} --n_epochs 4 --learning_rate_multiplier 0.05 --no_check_if_files_exist"""
+# print(command)
- # command = """python script.py"""
- process = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+# # command = """python script.py"""
+# process = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- while line := process.stdout.readline().decode().strip():
- if 'created fine-tune:' in line.lower():
- jobname = line.split()[-1]
- break
+# while line := process.stdout.readline().decode().strip():
+# if 'created fine-tune:' in line.lower():
+# jobname = line.split()[-1]
+# break
- # if 'costs $' in line.lower():
- # cost = line.split()[-1]
- # break
+# # if 'costs $' in line.lower():
+# # cost = line.split()[-1]
+# # break
- # assert cost[0] == '$'
+# # assert cost[0] == '$'
- # if float(cost[1:]) > 300:
- # print(f'Got cost {cost} -- you may wanna cancel the job: openai api fine_tunes.cancel -i {jobname}')
+# # if float(cost[1:]) > 300:
+# # print(f'Got cost {cost} -- you may wanna cancel the job: openai api fine_tunes.cancel -i {jobname}')
- # print(cost)
+# # print(cost)
- print(jobname)
+# print(jobname)
- # Block until it's done
- ft = openai_check_finetune(jobname)
- assert ft, ft
+# # Block until it's done
+# ft = openai_check_finetune(jobname)
+# assert ft, ft
- # Return its name
- return (jobname, ft)
+# # Return its name
+# return (jobname, ft)
-def openai_finetune(name, target):
- print(name)
- training_data_path = name_to_path(name)
- training_data_path += '.model'
+# def openai_finetune(name, target):
+# print(name)
+# training_data_path = name_to_path(name)
+# training_data_path += '.model'
- # if path + stuff exists, load the tuple from it
- try:
- with open(training_data_path) as f:
- jobname, ft = ujson.loads(f.readline())
+# # if path + stuff exists, load the tuple from it
+# try:
+# with open(training_data_path) as f:
+# jobname, ft = ujson.loads(f.readline())
- if openai_check_finetune(jobname):
- return jobname, ft
- except:
- pass
+# if openai_check_finetune(jobname):
+# return jobname, ft
+# except:
+# pass
- jobname, ft = openai_finetune_(name, target)
+# jobname, ft = openai_finetune_(name, target)
- with open(training_data_path, 'w') as f:
- f.write(ujson.dumps((jobname, ft)) + '\n')
+# with open(training_data_path, 'w') as f:
+# f.write(ujson.dumps((jobname, ft)) + '\n')
- return jobname, ft
+# return jobname, ft
-def name_to_path(name):
- if not os.path.exists(training_data_directory):
- os.makedirs(training_data_directory)
+# def name_to_path(name):
+# if not os.path.exists(training_data_directory):
+# os.makedirs(training_data_directory)
- training_data_path = os.path.join(training_data_directory, f'{name}.jsonl')
- return training_data_path
+# training_data_path = os.path.join(training_data_directory, f'{name}.jsonl')
+# return training_data_path
-# 3. Check that the output file name has status "success" (not deleted or non-existent). Otherwise, re-call with n = n+1.
-def finetune(training_data, target):
- name = Hasher.hash(training_data)
- training_data_path = name_to_path(name)
+# # 3. Check that the output file name has status "success" (not deleted or non-existent). Otherwise, re-call with n = n+1.
+# def finetune(training_data, target):
+# name = Hasher.hash(training_data)
+# training_data_path = name_to_path(name)
- with open(training_data_path, 'w') as f:
- for line in training_data:
- f.write(ujson.dumps(line) + '\n')
+# with open(training_data_path, 'w') as f:
+# for line in training_data:
+# f.write(ujson.dumps(line) + '\n')
- jobname, ft = openai_finetune(name, target)
- print(ft)
+# jobname, ft = openai_finetune(name, target)
+# print(ft)
- ft = dsp.GPT3(model=ft, stop=" </s>")
- return ft
+# ft = dsp.GPT3(model=ft, stop=" </s>")
+# return ft
-# 4. Return updated program.
-def compile(program, examples, target='ada'):
- training_data = simulate(program, examples)
- compiled_lm = finetune(training_data, target=target)
+# # 4. Return updated program.
+# def compile(program, examples, target='ada'):
+# training_data = simulate(program, examples)
+# compiled_lm = finetune(training_data, target=target)
- def compiled_program(*args, **kwargs):
- with dsp.settings.context(compiled_lm=compiled_lm, compiling=False):
- return program(*args, **kwargs)
+# def compiled_program(*args, **kwargs):
+# with dsp.settings.context(compiled_lm=compiled_lm, compiling=False):
+# return program(*args, **kwargs)
- compiled_program.lm = compiled_lm
- return compiled_program
+# compiled_program.lm = compiled_lm
+# return compiled_program
diff --git a/dsp/primitives/demonstrate.py b/dsp/primitives/demonstrate.py
index 215afc23d..33ba97932 100644
--- a/dsp/primitives/demonstrate.py
+++ b/dsp/primitives/demonstrate.py
@@ -45,62 +45,62 @@ class Example(dotdict):
return self.copy(demos=demos)
-def annotate(*transformations):
- """Returns an Augment function that applies the provided transformations to the Examples"""
+# def annotate(*transformations):
+# """Returns an Augment function that applies the provided transformations to the Examples"""
- def do_augment(train, k=None, return_all=False):
- rdemos = []
- ademos = []
+# def do_augment(train, k=None, return_all=False):
+# rdemos = []
+# ademos = []
- for example in train: # tqdm.tqdm
- raw_example = dsp.Example(example)
+# for example in train: # tqdm.tqdm
+# raw_example = dsp.Example(example)
- if (k is not None) and len(ademos) >= k:
- example = None
+# if (k is not None) and len(ademos) >= k:
+# example = None
- for f in transformations:
- if example is None:
- break
+# for f in transformations:
+# if example is None:
+# break
- example = f(example)
+# example = f(example)
- if example is not None:
- example.augmented = True
- ademos.append(example)
- else:
- raw_example.augmented = False
- rdemos.append(raw_example)
+# if example is not None:
+# example.augmented = True
+# ademos.append(example)
+# else:
+# raw_example.augmented = False
+# rdemos.append(raw_example)
- if return_all:
- return ademos + rdemos
+# if return_all:
+# return ademos + rdemos
- return ademos
+# return ademos
- return do_augment
+# return do_augment
-def sample(train: list[Example], k: int):
- """Sample k examples from train."""
- rng = random.Random(dsp.settings.branch_idx)
- shuffled_train = [dsp.Example(example) for example in train]
- rng.shuffle(shuffled_train)
+# def sample(train: list[Example], k: int):
+# """Sample k examples from train."""
+# rng = random.Random(dsp.settings.branch_idx)
+# shuffled_train = [dsp.Example(example) for example in train]
+# rng.shuffle(shuffled_train)
- return shuffled_train[:k]
+# return shuffled_train[:k]
-def all_but(train: list[Example], x: Example) -> list[Example]:
- """Removes the example x from the train set by comparing the question and history."""
+# def all_but(train: list[Example], x: Example) -> list[Example]:
+# """Removes the example x from the train set by comparing the question and history."""
- output = [
- y
- for y in train
- if not set.intersection(
- set(x.get("history", []) + [x.question]),
- set(y.get("history", []) + [y.question]),
- )
- ]
+# output = [
+# y
+# for y in train
+# if not set.intersection(
+# set(x.get("history", []) + [x.question]),
+# set(y.get("history", []) + [y.question]),
+# )
+# ]
- return output
+# return output
def passage_match(passages: list[str], answers: list[str]) -> bool:
diff --git a/dsp/primitives/inspect.py b/dsp/primitives/inspect.py
index ebcd3b787..edc8345d5 100644
--- a/dsp/primitives/inspect.py
+++ b/dsp/primitives/inspect.py
@@ -1,91 +1,91 @@
-import inspect
-import json
-import random
-import string
-
-import requests
-
-
-class FuncInspector:
- def __init__(self):
- self.calls = []
-
-
- def inspect_inner(self, func, function_calls):
- def wrapper(*args, **kwargs):
- result = func(*args, **kwargs)
- self.merge_result(result, function_calls)
- return result
- return wrapper
-
-
- def inspect_func(self, func):
- def wrapper(*args, **kwargs):
- result = func(*args, **kwargs)
- stack = inspect.stack()
- function_calls = []
- for i in range(len(stack)):
- if stack[i][3] == "<module>":
- break
- if stack[i][3] != "wrapper":
- function_calls.append(stack[i][3])
- function_calls.reverse()
- result = self.inspect_inner(result, function_calls)
- return result
- return wrapper
+# import inspect
+# import json
+# import random
+# import string
+
+# import requests
+
+
+# class FuncInspector:
+# def __init__(self):
+# self.calls = []
+
+
+# def inspect_inner(self, func, function_calls):
+# def wrapper(*args, **kwargs):
+# result = func(*args, **kwargs)
+# self.merge_result(result, function_calls)
+# return result
+# return wrapper
+
+
+# def inspect_func(self, func):
+# def wrapper(*args, **kwargs):
+# result = func(*args, **kwargs)
+# stack = inspect.stack()
+# function_calls = []
+# for i in range(len(stack)):
+# if stack[i][3] == "<module>":
+# break
+# if stack[i][3] != "wrapper":
+# function_calls.append(stack[i][3])
+# function_calls.reverse()
+# result = self.inspect_inner(result, function_calls)
+# return result
+# return wrapper
- def parse(self, obj, delete_empty=False):
- if isinstance(obj, list):
- for elem in obj:
- self.parse(elem, delete_empty)
- if isinstance(obj, dict):
- to_delete = []
- for key in obj:
- if delete_empty and not obj[key] or key == "completions":
- to_delete.append(key)
- else:
- self.parse(obj[key], delete_empty)
- for key in to_delete:
- obj.pop(key)
-
-
- def merge_result(self, result, function_calls):
- prev_list = self.calls
- prev_call = {} if not prev_list else prev_list[-1]
- for call in function_calls[:-1]:
- if call not in prev_call:
- prev_call = {call: []}
- prev_list.append(prev_call)
- prev_list = prev_call[call]
- prev_call = {} if not prev_list else prev_list[-1]
-
- example_obj = result[0]
- self.parse(example_obj)
- prev_list.append({ function_calls[-1]: example_obj })
-
-
- def view_data(self):
- chars = string.digits + string.ascii_lowercase
- id = ''.join(random.choices(chars, k=8))
-
- post_url = 'http://127.0.0.1:5000/log-item'
- parsed_calls = self.calls.copy()
- self.parse(parsed_calls, delete_empty=True)
- data = {'id': id, 'content': parsed_calls}
- response = requests.post(post_url, json=data)
+# def parse(self, obj, delete_empty=False):
+# if isinstance(obj, list):
+# for elem in obj:
+# self.parse(elem, delete_empty)
+# if isinstance(obj, dict):
+# to_delete = []
+# for key in obj:
+# if delete_empty and not obj[key] or key == "completions":
+# to_delete.append(key)
+# else:
+# self.parse(obj[key], delete_empty)
+# for key in to_delete:
+# obj.pop(key)
+
+
+# def merge_result(self, result, function_calls):
+# prev_list = self.calls
+# prev_call = {} if not prev_list else prev_list[-1]
+# for call in function_calls[:-1]:
+# if call not in prev_call:
+# prev_call = {call: []}
+# prev_list.append(prev_call)
+# prev_list = prev_call[call]
+# prev_call = {} if not prev_list else prev_list[-1]
+
+# example_obj = result[0]
+# self.parse(example_obj)
+# prev_list.append({ function_calls[-1]: example_obj })
+
+
+# def view_data(self):
+# chars = string.digits + string.ascii_lowercase
+# id = ''.join(random.choices(chars, k=8))
+
+# post_url = 'http://127.0.0.1:5000/log-item'
+# parsed_calls = self.calls.copy()
+# self.parse(parsed_calls, delete_empty=True)
+# data = {'id': id, 'content': parsed_calls}
+# response = requests.post(post_url, json=data)
- if response.status_code == 201:
- print('Data created successfully')
- else:
- print(f'Error sending data to server: {response.status_code}')
- return
+# if response.status_code == 201:
+# print('Data created successfully')
+# else:
+# print(f'Error sending data to server: {response.status_code}')
+# return
- frontend_url = f"http://localhost:3000?id={id}"
- print(f"View the data here, {frontend_url}")
+# frontend_url = f"http://localhost:3000?id={id}"
+# print(f"View the data here, {frontend_url}")
- def output_json(self, out_path):
- f = open(out_path, "w")
- json_object = json.dumps(self.calls, indent=2)
- f.write(json_object)
+# def output_json(self, out_path):
+# f = open(out_path, "w")
+# json_object = json.dumps(self.calls, indent=2)
+# f.write(json_object)
diff --git a/dsp/primitives/primitives.py b/dsp/primitives/primitives.py
index 2e4c4bfc0..50aea522b 100644
--- a/dsp/primitives/primitives.py
+++ b/dsp/primitives/primitives.py
@@ -1,46 +1,46 @@
-from functools import wraps
+# from functools import wraps
-import dsp
+# import dsp
-# applied right to left (innermost first, like function calls)
-def compose_decorators(*decorators):
- def decorator(func):
- for decorator in decorators[::-1]:
- func = decorator(func)
- return func
- return decorator
+# # applied right to left (innermost first, like function calls)
+# def compose_decorators(*decorators):
+# def decorator(func):
+# for decorator in decorators[::-1]:
+# func = decorator(func)
+# return func
+# return decorator
-def shallow_copy_example_args(func):
- @wraps(func)
- def wrapper(*args, **kwargs):
- args = [dsp.Example(arg) if isinstance(arg, dsp.Example) else arg for arg in args]
- kwargs = {key: dsp.Example(value) if isinstance(value, dsp.Example) else value for key, value in kwargs.items()}
- return func(*args, **kwargs)
- return wrapper
+# def shallow_copy_example_args(func):
+# @wraps(func)
+# def wrapper(*args, **kwargs):
+# args = [dsp.Example(arg) if isinstance(arg, dsp.Example) else arg for arg in args]
+# kwargs = {key: dsp.Example(value) if isinstance(value, dsp.Example) else value for key, value in kwargs.items()}
+# return func(*args, **kwargs)
+# return wrapper
-transformation = shallow_copy_example_args
-# transformation = compose_decorators(handle_compilation, shallow_copy_example_args)
+# transformation = shallow_copy_example_args
+# # transformation = compose_decorators(handle_compilation, shallow_copy_example_args)
-def compiled(func):
- def wrapper(*args, **kwargs):
- is_to_be_compiled = True #decorator_kwargs.get('compile', False)
- compiled_lm = dsp.settings.compiled_lm
+# def compiled(func):
+# def wrapper(*args, **kwargs):
+# is_to_be_compiled = True #decorator_kwargs.get('compile', False)
+# compiled_lm = dsp.settings.compiled_lm
- if is_to_be_compiled and compiled_lm:
- assert len(args) == 1, len(args)
- example = args[0]
+# if is_to_be_compiled and compiled_lm:
+# assert len(args) == 1, len(args)
+# example = args[0]
- with dsp.settings.context(lm=compiled_lm, show_guidelines=False):
- old_demos = list(example.demos)
- example = func(example.copy(demos=[]), **kwargs)
- return example.copy(demos=old_demos)
+# with dsp.settings.context(lm=compiled_lm, show_guidelines=False):
+# old_demos = list(example.demos)
+# example = func(example.copy(demos=[]), **kwargs)
+# return example.copy(demos=old_demos)
- with dsp.settings.context(compiling=True):
- return func(*args, **kwargs)
+# with dsp.settings.context(compiling=True):
+# return func(*args, **kwargs)
- return wrapper
+# return wrapper
diff --git a/dsp/templates/utils.py b/dsp/templates/utils.py
index 53624ad2d..4bc15c058 100644
--- a/dsp/templates/utils.py
+++ b/dsp/templates/utils.py
@@ -17,29 +17,29 @@ def passages2text(passages: Union[str, list, tuple]) -> str:
return "\n".join([f"[{idx+1}] «{txt}»" for idx, txt in enumerate(passages)])
-def passages2textV2(passages: Union[str, list, tuple]) -> str:
- """Formats the given one or more passages into a single structured string."""
- if isinstance(passages, str):
- return passages
-
- assert type(passages) in [list, tuple]
-
- def psg2text(psg):
- try:
- title, snippet = psg.split("|", 1)
- return f"Title: {title.strip()} | Snippet: «{snippet.strip()}»"
- except Exception:
- pass
+# def passages2textV2(passages: Union[str, list, tuple]) -> str:
+# """Formats the given one or more passages into a single structured string."""
+# if isinstance(passages, str):
+# return passages
+
+# assert type(passages) in [list, tuple]
+
+# def psg2text(psg):
+# try:
+# title, snippet = psg.split("|", 1)
+# return f"Title: {title.strip()} | Snippet: «{snippet.strip()}»"
+# except Exception:
+# pass
- return f"«{psg}»"
+# return f"«{psg}»"
- if len(passages) == 0:
- return "N/A"
+# if len(passages) == 0:
+# return "N/A"
- if len(passages) == 1:
- return psg2text(passages[0])
+# if len(passages) == 1:
+# return psg2text(passages[0])
- return "\n".join([f"[{idx+1}] {psg2text(txt)}" for idx, txt in enumerate(passages)])
+# return "\n".join([f"[{idx+1}] {psg2text(txt)}" for idx, txt in enumerate(passages)])
def format_answers(answers: Union[str, list]) -> Optional[str]: