DSPy: The framework for programming—not prompting—language models
Python
Adding new dspy.LM and dspy.Adapter
Omar Khattab07ae49d4e51f
diff --git a/.gitignore b/.gitignoreindex fbbe65d2a..d34d0c34e 100644--- a/.gitignore+++ b/.gitignore@@ -56,3 +56,4 @@ assertion.log/.devcontainer/.personalization.sh.mypy_cache+dummy.csv\ No newline at end of filediff --git a/dspy/__init__.py b/dspy/__init__.pyindex 78ae04b03..9919fc7dc 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -9,10 +9,12 @@ from .utils.logging import logger, set_log_output# Functional must be imported after primitives, predict and signaturesfrom .functional import * # isort: skip+from dspy.evaluate import Evaluate # isort: skip+from dspy.clients import * # isort: skipsettings = dsp.settings-LM = dsp.LM+# LM = dsp.LMAzureOpenAI = dsp.AzureOpenAIOpenAI = dsp.GPT3diff --git a/dspy/adapters/__init__.py b/dspy/adapters/__init__.pyindex e69de29bb..85cac40b2 100644--- a/dspy/adapters/__init__.py+++ b/dspy/adapters/__init__.py@@ -0,0 +1,2 @@+from .base import Adapter+from .chat_adapter import ChatAdapter\ No newline at end of filediff --git a/dspy/adapters/base.py b/dspy/adapters/base.pynew file mode 100644index 000000000..5010a8c76--- /dev/null+++ b/dspy/adapters/base.py@@ -0,0 +1,12 @@+class Adapter:+ def __call__(self, lm, lm_kwargs, signature, demos, inputs):+ messages = self.format(signature, demos, inputs)+ outputs = lm(messages=messages, **lm_kwargs)+ values = []++ for output in outputs:+ value = self.parse(signature, output)+ assert set(value.keys()) == set(signature.output_fields.keys()), f"Expected {signature.output_fields.keys()} but got {value.keys()}"+ values.append(value)++ return values\ No newline at end of filediff --git a/dspy/adapters/basic_adapter.py b/dspy/adapters/basic_adapter.pydeleted file mode 100644index e69de29bb..000000000diff --git a/dspy/adapters/chat_adapter.py b/dspy/adapters/chat_adapter.pynew file mode 100644index 000000000..f7928ed17--- /dev/null+++ b/dspy/adapters/chat_adapter.py@@ -0,0 +1,80 @@+import re+from .base import Adapter++field_header_pattern = re.compile(r'\[\[\[\[ #### (\w+) #### \]\]\]\]')+++class ChatAdapter(Adapter):+ def __init__(self):+ pass++ def format(self, signature, demos, inputs):+ messages = []+ messages.append({"role": "system", "content": prepare_instructions(signature)})++ for demo in demos:+ output_fields_, demo_ = list(signature.output_fields.keys()) + ['completed'], {**demo, 'completed': ''}+ messages.append({"role": "user", "content": format_chat_turn(signature.input_fields.keys(), demo)})+ messages.append({"role": "assistant", "content": format_chat_turn(output_fields_, demo_)})++ messages.append({"role": "user", "content": format_chat_turn(signature.input_fields.keys(), inputs)})++ return messages++ def parse(self, signature, completion):+ sections = [(None, [])]++ for line in completion.splitlines():+ match = field_header_pattern.match(line.strip())+ if match: sections.append((match.group(1), []))+ else: sections[-1][1].append(line)++ sections = [(k, '\n'.join(v).strip()) for k, v in sections]++ fields = {}+ for k, v in sections:+ if (k not in fields) and (k in signature.output_fields): fields[k] = v++ if fields.keys() != signature.output_fields.keys():+ raise ValueError(f"Expected {signature.output_fields.keys()} but got {fields.keys()}")++ return fields+++def format_fields(fields):+ return '\n\n'.join([f"[[[[ #### {k} #### ]]]]\n{v}" for k, v in fields.items()]).strip()++def format_chat_turn(field_names, values):+ if not set(values).issuperset(set(field_names)):+ raise ValueError(f"Expected {field_names} but got {values.keys()}")++ return format_fields({k: values[k] for k in field_names})++def enumerate_fields(fields):+ parts = []+ for idx, (k, v) in enumerate(fields.items()):+ parts.append(f"{idx+1}. `{k}`")+ parts[-1] += f" ({v.annotation.__name__})"+ parts[-1] += f": {v.json_schema_extra['desc']}" if v.json_schema_extra['desc'] != f'${{{k}}}' else ''++ return '\n'.join(parts).strip()++def prepare_instructions(signature):+ parts = []+ parts.append("Your input fields are:\n" + enumerate_fields(signature.input_fields))+ parts.append("Your output fields are:\n" + enumerate_fields(signature.output_fields))+ parts.append("All interactions will be structured in the following way, with the appropriate values filled in.")++ parts.append(format_fields({f : f"{{{f}}}" for f in signature.input_fields}))+ parts.append(format_fields({f : f"{{{f}}}" for f in signature.output_fields}))+ parts.append(format_fields({'completed' : ""}))++ parts.append("You will receive some input fields in each interaction. " ++ "Respond only with the corresponding output fields, starting with the field " ++ ", then ".join(f"`{f}`" for f in signature.output_fields) ++ ", and then ending with the marker for `completed`.")++ objective = ('\n' + ' ' * 8).join([''] + signature.instructions.splitlines())+ parts.append(f"In adhering to this structure, your objective is: {objective}")++ return '\n\n'.join(parts).strip()diff --git a/dspy/adapters/chatml_adapter.py b/dspy/adapters/chatml_adapter.pydeleted file mode 100644index e69de29bb..000000000diff --git a/dspy/adapters/llamachat_adapter.py b/dspy/adapters/llamachat_adapter.pydeleted file mode 100644index e69de29bb..000000000diff --git a/dspy/adapters/vicuna_adapter.py b/dspy/adapters/vicuna_adapter.pydeleted file mode 100644index e69de29bb..000000000diff --git a/dspy/clients/__init__.py b/dspy/clients/__init__.pynew file mode 100644index 000000000..079dc5420--- /dev/null+++ b/dspy/clients/__init__.py@@ -0,0 +1 @@+from .lm import LM\ No newline at end of filediff --git a/dspy/clients/lm.py b/dspy/clients/lm.pynew file mode 100644index 000000000..43ba3a784--- /dev/null+++ b/dspy/clients/lm.py@@ -0,0 +1,98 @@+import os+import ujson+import litellm+import functools++from litellm.caching import Cache+litellm.cache = Cache(disk_cache_dir=".dspy_cache", type="disk")++class LM:+ def __init__(self, model, model_type='chat', temperature=0.0, cache=True, **kwargs):+ self.model = model+ self.model_type = model_type+ self.cache = cache+ self.kwargs = dict(temperature=temperature, **kwargs)+ self.history = []++ def __call__(self, prompt=None, messages=None, **kwargs):+ # Build the request.+ cache = kwargs.pop("cache", self.cache)+ messages = messages or [{"role": "user", "content": prompt}]+ kwargs = {**self.kwargs, **kwargs}++ # Make the request and handle LRU & disk caching.+ if self.model_type == "chat": completion = cached_litellm_completion if cache else litellm_completion+ else: completion = cached_litellm_text_completion if cache else litellm_text_completion++ response = completion(ujson.dumps(dict(model=self.model, messages=messages, **kwargs)))+ outputs = [c.message.content if hasattr(c, "message") else c["text"] for c in response.choices]++ # Logging, with removed api key & where `cost` is None on cache hit.+ kwargs = {k: v for k, v in kwargs.items() if not k.startswith("api_")}+ entry = dict(prompt=prompt, messages=messages, kwargs=kwargs, response=response, outputs=outputs)+ entry = dict(**entry, usage=dict(response["usage"]), cost=response["_hidden_params"].get("response_cost"))+ self.history.append(entry)++ return outputs++ def inspect_history(self, n: int = 1):+ _inspect_history(self, n)++[email protected]_cache(maxsize=None)+def cached_litellm_completion(request):+ return litellm_completion(request, cache={"no-cache": False, "no-store": False})++def litellm_completion(request, cache={"no-cache": True, "no-store": True}):+ kwargs = ujson.loads(request)+ return litellm.completion(cache=cache, **kwargs)+[email protected]_cache(maxsize=None)+def cached_litellm_text_completion(request):+ return litellm_text_completion(request, cache=None)++def litellm_text_completion(request, cache={"no-cache": True, "no-store": True}):+ kwargs = ujson.loads(request)++ # Extract the provider and model from the model string.+ model = kwargs.pop("model").split("/", 1)+ provider, model = model[0] if len(model) > 1 else "openai", model[-1]++ # Use the API key and base from the kwargs, or from the environment.+ api_key = kwargs.pop("api_key", None) or os.getenv(f"{provider}_API_KEY")+ api_base = kwargs.pop("api_base", None) or os.getenv(f"{provider}_API_BASE")++ # Build the prompt from the messages.+ prompt = '\n\n'.join([x['content'] for x in kwargs.pop("messages")] + ['BEGIN RESPONSE:'])++ return litellm.text_completion(cache=cache, model=f'text-completion-openai/{model}', api_key=api_key,+ api_base=api_base, prompt=prompt, **kwargs)+++def _green(text: str, end: str = "\n"):+ return "\x1b[32m" + str(text).lstrip() + "\x1b[0m" + end++def _red(text: str, end: str = "\n"):+ return "\x1b[31m" + str(text) + "\x1b[0m" + end++def _inspect_history(lm, n: int = 1):+ """Prints the last n prompts and their completions."""++ for item in reversed(lm.history[-n:]):+ messages = item["messages"] or [{"role": "user", "content": item['prompt']}]+ outputs = item["outputs"]++ print("\n\n\n")+ for msg in messages:+ print(_red(f"{msg['role'].capitalize()} message:"))+ print(msg['content'].strip())+ print("\n")++ print(_red(f"Response:"))+ print(_green(outputs[0].strip()))++ if len(outputs) > 1:+ choices_text = f" \t (and {len(outputs)-1} other completions)"+ print(_red(choices_text, end=""))++ print("\n\n\n")diff --git a/dspy/predict/predict.py b/dspy/predict/predict.pyindex 8e72d897f..a7e5b7309 100644--- a/dspy/predict/predict.py+++ b/dspy/predict/predict.py@@ -106,14 +106,10 @@ class Predict(Module, Parameter):# If temperature is 0.0 but its n > 1, set temperature to 0.7.temperature = config.get("temperature")temperature = lm.kwargs["temperature"] if temperature is None else temperature-- num_generations = config.get("n")- if num_generations is None:- num_generations = lm.kwargs.get("n", lm.kwargs.get("num_generations", 1))+ num_generations = config.get("n") or lm.kwargs.get("n") or lm.kwargs.get("num_generations") or 1if (temperature is None or temperature <= 0.15) and num_generations > 1:config["temperature"] = 0.7- # print(f"#> Setting temperature to 0.7 since n={num_generations} and prior temperature={temperature}.")if new_signature is not None:signature = new_signature@@ -123,7 +119,10 @@ class Predict(Module, Parameter):missing = [k for k in signature.input_fields if k not in kwargs]print(f"WARNING: Not all input fields were provided to module. Present: {present}. Missing: {missing}.")- if dsp.settings.experimental:+ import dspy+ if isinstance(lm, dspy.LM):+ completions = v2_5_generate(lm, config, signature, demos, kwargs)+ elif dsp.settings.experimental:completions = new_generate(lm, signature, dsp.Example(demos=demos, **kwargs), **config)else:completions = old_generate(demos, signature, kwargs, config, self.lm, self.stage)@@ -209,6 +208,15 @@ def new_generate(lm, signature, example, max_depth=6, **kwargs):return completions++def v2_5_generate(lm, lm_kwargs, signature, demos, inputs):+ from dspy.adapters import ChatAdapter++ adapter = ChatAdapter()+ return adapter(lm, lm_kwargs=lm_kwargs, signature=signature, demos=demos, inputs=inputs)+++# TODO: get some defaults during init from the context window?# # TODO: FIXME: Hmm, I guess expected behavior is that contexts can# affect execution. Well, we need to determine whether context dominates, __init__ demoninates, or forward dominates.diff --git a/dspy/predict/react.py b/dspy/predict/react.pyindex 3a2e5d155..c86127bae 100644--- a/dspy/predict/react.py+++ b/dspy/predict/react.py@@ -10,6 +10,8 @@ from .predict import Predict# TODO: There's a lot of value in having a stopping condition in the LM calls at `\n\nObservation:`+# TODO [NEW]: When max_iters is about to be reached, reduce the set of available actions to only the Finish action.+class ReAct(Module):def __init__(self, signature, max_iters=5, num_results=3, tools=None):diff --git a/tests/modules/test_cloudflare_models.py b/tests/modules/test_cloudflare_models.pyindex c2c99b428..2ab222c86 100644--- a/tests/modules/test_cloudflare_models.py+++ b/tests/modules/test_cloudflare_models.py@@ -36,7 +36,7 @@ models = {}-def get_lm(name: str) -> dspy.LM:+def get_lm(name: str): # -> dspy.LM:return dspy.CloudflareAI(model=name)