Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Adding new dspy.LM and dspy.Adapter

Omar Khattab07ae49d4e51f
diff --git a/.gitignore b/.gitignore
index fbbe65d2a..d34d0c34e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -56,3 +56,4 @@ assertion.log
/.devcontainer/.personalization.sh
.mypy_cache
+dummy.csv
\ No newline at end of file
diff --git a/dspy/__init__.py b/dspy/__init__.py
index 78ae04b03..9919fc7dc 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -9,10 +9,12 @@ from .utils.logging import logger, set_log_output
# Functional must be imported after primitives, predict and signatures
from .functional import * # isort: skip
+from dspy.evaluate import Evaluate # isort: skip
+from dspy.clients import * # isort: skip
settings = dsp.settings
-LM = dsp.LM
+# LM = dsp.LM
AzureOpenAI = dsp.AzureOpenAI
OpenAI = dsp.GPT3
diff --git a/dspy/adapters/__init__.py b/dspy/adapters/__init__.py
index e69de29bb..85cac40b2 100644
--- a/dspy/adapters/__init__.py
+++ b/dspy/adapters/__init__.py
@@ -0,0 +1,2 @@
+from .base import Adapter
+from .chat_adapter import ChatAdapter
\ No newline at end of file
diff --git a/dspy/adapters/base.py b/dspy/adapters/base.py
new file mode 100644
index 000000000..5010a8c76
--- /dev/null
+++ b/dspy/adapters/base.py
@@ -0,0 +1,12 @@
+class Adapter:
+ def __call__(self, lm, lm_kwargs, signature, demos, inputs):
+ messages = self.format(signature, demos, inputs)
+ outputs = lm(messages=messages, **lm_kwargs)
+ values = []
+
+ for output in outputs:
+ value = self.parse(signature, output)
+ assert set(value.keys()) == set(signature.output_fields.keys()), f"Expected {signature.output_fields.keys()} but got {value.keys()}"
+ values.append(value)
+
+ return values
\ No newline at end of file
diff --git a/dspy/adapters/basic_adapter.py b/dspy/adapters/basic_adapter.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/dspy/adapters/chat_adapter.py b/dspy/adapters/chat_adapter.py
new file mode 100644
index 000000000..f7928ed17
--- /dev/null
+++ b/dspy/adapters/chat_adapter.py
@@ -0,0 +1,80 @@
+import re
+from .base import Adapter
+
+field_header_pattern = re.compile(r'\[\[\[\[ #### (\w+) #### \]\]\]\]')
+
+
+class ChatAdapter(Adapter):
+ def __init__(self):
+ pass
+
+ def format(self, signature, demos, inputs):
+ messages = []
+ messages.append({"role": "system", "content": prepare_instructions(signature)})
+
+ for demo in demos:
+ output_fields_, demo_ = list(signature.output_fields.keys()) + ['completed'], {**demo, 'completed': ''}
+ messages.append({"role": "user", "content": format_chat_turn(signature.input_fields.keys(), demo)})
+ messages.append({"role": "assistant", "content": format_chat_turn(output_fields_, demo_)})
+
+ messages.append({"role": "user", "content": format_chat_turn(signature.input_fields.keys(), inputs)})
+
+ return messages
+
+ def parse(self, signature, completion):
+ sections = [(None, [])]
+
+ for line in completion.splitlines():
+ match = field_header_pattern.match(line.strip())
+ if match: sections.append((match.group(1), []))
+ else: sections[-1][1].append(line)
+
+ sections = [(k, '\n'.join(v).strip()) for k, v in sections]
+
+ fields = {}
+ for k, v in sections:
+ if (k not in fields) and (k in signature.output_fields): fields[k] = v
+
+ if fields.keys() != signature.output_fields.keys():
+ raise ValueError(f"Expected {signature.output_fields.keys()} but got {fields.keys()}")
+
+ return fields
+
+
+def format_fields(fields):
+ return '\n\n'.join([f"[[[[ #### {k} #### ]]]]\n{v}" for k, v in fields.items()]).strip()
+
+def format_chat_turn(field_names, values):
+ if not set(values).issuperset(set(field_names)):
+ raise ValueError(f"Expected {field_names} but got {values.keys()}")
+
+ return format_fields({k: values[k] for k in field_names})
+
+def enumerate_fields(fields):
+ parts = []
+ for idx, (k, v) in enumerate(fields.items()):
+ parts.append(f"{idx+1}. `{k}`")
+ parts[-1] += f" ({v.annotation.__name__})"
+ parts[-1] += f": {v.json_schema_extra['desc']}" if v.json_schema_extra['desc'] != f'${{{k}}}' else ''
+
+ return '\n'.join(parts).strip()
+
+def prepare_instructions(signature):
+ parts = []
+ parts.append("Your input fields are:\n" + enumerate_fields(signature.input_fields))
+ parts.append("Your output fields are:\n" + enumerate_fields(signature.output_fields))
+ parts.append("All interactions will be structured in the following way, with the appropriate values filled in.")
+
+ parts.append(format_fields({f : f"{{{f}}}" for f in signature.input_fields}))
+ parts.append(format_fields({f : f"{{{f}}}" for f in signature.output_fields}))
+ parts.append(format_fields({'completed' : ""}))
+
+ parts.append("You will receive some input fields in each interaction. " +
+ "Respond only with the corresponding output fields, starting with the field " +
+ ", then ".join(f"`{f}`" for f in signature.output_fields) +
+ ", and then ending with the marker for `completed`.")
+
+ objective = ('\n' + ' ' * 8).join([''] + signature.instructions.splitlines())
+ parts.append(f"In adhering to this structure, your objective is: {objective}")
+
+ return '\n\n'.join(parts).strip()
diff --git a/dspy/adapters/chatml_adapter.py b/dspy/adapters/chatml_adapter.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/dspy/adapters/llamachat_adapter.py b/dspy/adapters/llamachat_adapter.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/dspy/adapters/vicuna_adapter.py b/dspy/adapters/vicuna_adapter.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/dspy/clients/__init__.py b/dspy/clients/__init__.py
new file mode 100644
index 000000000..079dc5420
--- /dev/null
+++ b/dspy/clients/__init__.py
@@ -0,0 +1 @@
+from .lm import LM
\ No newline at end of file
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
new file mode 100644
index 000000000..43ba3a784
--- /dev/null
+++ b/dspy/clients/lm.py
@@ -0,0 +1,98 @@
+import os
+import ujson
+import litellm
+import functools
+
+from litellm.caching import Cache
+litellm.cache = Cache(disk_cache_dir=".dspy_cache", type="disk")
+
+class LM:
+ def __init__(self, model, model_type='chat', temperature=0.0, cache=True, **kwargs):
+ self.model = model
+ self.model_type = model_type
+ self.cache = cache
+ self.kwargs = dict(temperature=temperature, **kwargs)
+ self.history = []
+
+ def __call__(self, prompt=None, messages=None, **kwargs):
+ # Build the request.
+ cache = kwargs.pop("cache", self.cache)
+ messages = messages or [{"role": "user", "content": prompt}]
+ kwargs = {**self.kwargs, **kwargs}
+
+ # Make the request and handle LRU & disk caching.
+ if self.model_type == "chat": completion = cached_litellm_completion if cache else litellm_completion
+ else: completion = cached_litellm_text_completion if cache else litellm_text_completion
+
+ response = completion(ujson.dumps(dict(model=self.model, messages=messages, **kwargs)))
+ outputs = [c.message.content if hasattr(c, "message") else c["text"] for c in response.choices]
+
+ # Logging, with removed api key & where `cost` is None on cache hit.
+ kwargs = {k: v for k, v in kwargs.items() if not k.startswith("api_")}
+ entry = dict(prompt=prompt, messages=messages, kwargs=kwargs, response=response, outputs=outputs)
+ entry = dict(**entry, usage=dict(response["usage"]), cost=response["_hidden_params"].get("response_cost"))
+ self.history.append(entry)
+
+ return outputs
+
+ def inspect_history(self, n: int = 1):
+ _inspect_history(self, n)
+
+
[email protected]_cache(maxsize=None)
+def cached_litellm_completion(request):
+ return litellm_completion(request, cache={"no-cache": False, "no-store": False})
+
+def litellm_completion(request, cache={"no-cache": True, "no-store": True}):
+ kwargs = ujson.loads(request)
+ return litellm.completion(cache=cache, **kwargs)
+
[email protected]_cache(maxsize=None)
+def cached_litellm_text_completion(request):
+ return litellm_text_completion(request, cache=None)
+
+def litellm_text_completion(request, cache={"no-cache": True, "no-store": True}):
+ kwargs = ujson.loads(request)
+
+ # Extract the provider and model from the model string.
+ model = kwargs.pop("model").split("/", 1)
+ provider, model = model[0] if len(model) > 1 else "openai", model[-1]
+
+ # Use the API key and base from the kwargs, or from the environment.
+ api_key = kwargs.pop("api_key", None) or os.getenv(f"{provider}_API_KEY")
+ api_base = kwargs.pop("api_base", None) or os.getenv(f"{provider}_API_BASE")
+
+ # Build the prompt from the messages.
+ prompt = '\n\n'.join([x['content'] for x in kwargs.pop("messages")] + ['BEGIN RESPONSE:'])
+
+ return litellm.text_completion(cache=cache, model=f'text-completion-openai/{model}', api_key=api_key,
+ api_base=api_base, prompt=prompt, **kwargs)
+
+
+def _green(text: str, end: str = "\n"):
+ return "\x1b[32m" + str(text).lstrip() + "\x1b[0m" + end
+
+def _red(text: str, end: str = "\n"):
+ return "\x1b[31m" + str(text) + "\x1b[0m" + end
+
+def _inspect_history(lm, n: int = 1):
+ """Prints the last n prompts and their completions."""
+
+ for item in reversed(lm.history[-n:]):
+ messages = item["messages"] or [{"role": "user", "content": item['prompt']}]
+ outputs = item["outputs"]
+
+ print("\n\n\n")
+ for msg in messages:
+ print(_red(f"{msg['role'].capitalize()} message:"))
+ print(msg['content'].strip())
+ print("\n")
+
+ print(_red(f"Response:"))
+ print(_green(outputs[0].strip()))
+
+ if len(outputs) > 1:
+ choices_text = f" \t (and {len(outputs)-1} other completions)"
+ print(_red(choices_text, end=""))
+
+ print("\n\n\n")
diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py
index 8e72d897f..a7e5b7309 100644
--- a/dspy/predict/predict.py
+++ b/dspy/predict/predict.py
@@ -106,14 +106,10 @@ class Predict(Module, Parameter):
# If temperature is 0.0 but its n > 1, set temperature to 0.7.
temperature = config.get("temperature")
temperature = lm.kwargs["temperature"] if temperature is None else temperature
-
- num_generations = config.get("n")
- if num_generations is None:
- num_generations = lm.kwargs.get("n", lm.kwargs.get("num_generations", 1))
+ num_generations = config.get("n") or lm.kwargs.get("n") or lm.kwargs.get("num_generations") or 1
if (temperature is None or temperature <= 0.15) and num_generations > 1:
config["temperature"] = 0.7
- # print(f"#> Setting temperature to 0.7 since n={num_generations} and prior temperature={temperature}.")
if new_signature is not None:
signature = new_signature
@@ -123,7 +119,10 @@ class Predict(Module, Parameter):
missing = [k for k in signature.input_fields if k not in kwargs]
print(f"WARNING: Not all input fields were provided to module. Present: {present}. Missing: {missing}.")
- if dsp.settings.experimental:
+ import dspy
+ if isinstance(lm, dspy.LM):
+ completions = v2_5_generate(lm, config, signature, demos, kwargs)
+ elif dsp.settings.experimental:
completions = new_generate(lm, signature, dsp.Example(demos=demos, **kwargs), **config)
else:
completions = old_generate(demos, signature, kwargs, config, self.lm, self.stage)
@@ -209,6 +208,15 @@ def new_generate(lm, signature, example, max_depth=6, **kwargs):
return completions
+
+def v2_5_generate(lm, lm_kwargs, signature, demos, inputs):
+ from dspy.adapters import ChatAdapter
+
+ adapter = ChatAdapter()
+ return adapter(lm, lm_kwargs=lm_kwargs, signature=signature, demos=demos, inputs=inputs)
+
+
+
# TODO: get some defaults during init from the context window?
# # TODO: FIXME: Hmm, I guess expected behavior is that contexts can
# affect execution. Well, we need to determine whether context dominates, __init__ demoninates, or forward dominates.
diff --git a/dspy/predict/react.py b/dspy/predict/react.py
index 3a2e5d155..c86127bae 100644
--- a/dspy/predict/react.py
+++ b/dspy/predict/react.py
@@ -10,6 +10,8 @@ from .predict import Predict
# TODO: There's a lot of value in having a stopping condition in the LM calls at `\n\nObservation:`
+# TODO [NEW]: When max_iters is about to be reached, reduce the set of available actions to only the Finish action.
+
class ReAct(Module):
def __init__(self, signature, max_iters=5, num_results=3, tools=None):
diff --git a/tests/modules/test_cloudflare_models.py b/tests/modules/test_cloudflare_models.py
index c2c99b428..2ab222c86 100644
--- a/tests/modules/test_cloudflare_models.py
+++ b/tests/modules/test_cloudflare_models.py
@@ -36,7 +36,7 @@ models = {
}
-def get_lm(name: str) -> dspy.LM:
+def get_lm(name: str): # -> dspy.LM:
return dspy.CloudflareAI(model=name)