DSPy: The framework for programming—not prompting—language models
Python
Addressed Llamma Cpp Support PR Comments
Abdul Dridiac94c15cc8b4
diff --git a/docs/api/local_language_model_clients/LlamaCpp.md b/docs/api/local_language_model_clients/LlamaCpp.mdnew file mode 100644index 000000000..699505f70--- /dev/null+++ b/docs/api/local_language_model_clients/LlamaCpp.md@@ -0,0 +1,42 @@+# dspy.LlamaCpp++## Prerequisites++Install Llama Cpp Python by following the instructions provided in the [Llama Cpp Python repository](https://github.com/abetlen/llama-cpp-python).++```shell+pip install llama-cpp-python+```++alternatively, to install with CUDA support:++```shell+CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python+```+++### Initializing the Llama model++Initialize the model within your program with the desired parameters.++```python+from llama_cpp import Llama++llm = Llama(+ model_path="./sppo_finetuned_llama_3_8b.gguf",+ n_gpu_layers=-1,+ n_ctx=0,+ verbose=False+)+```+++### Sending requests to the model++After initializing the Llama model, you can interact with it using the `LlamaCpp` client.++```python+import dspy++llamalm = dspy.LlamaCpp(model="llama", llama_model=llm, model_type="chat", temperature=0.4)+```\ No newline at end of filediff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.pyindex 8428db4b2..c0aa98cd8 100644--- a/dsp/modules/__init__.py+++ b/dsp/modules/__init__.py@@ -18,6 +18,7 @@ from .gpt3 import *from .groq_client import *from .hf import HFModelfrom .hf_client import Anyscale, HFClientTGI, Together+from .llama import *from .mistral import *from .ollama import *from .premai import PremAIdiff --git a/dsp/modules/llama.py b/dsp/modules/llama.pynew file mode 100644index 000000000..e1a920eac--- /dev/null+++ b/dsp/modules/llama.py@@ -0,0 +1,99 @@+try:+ import llama_cpp+ from llama_cpp import Llama+except ImportError as exc:+ raise ModuleNotFoundError(+ """You need to install llama_cpp library to use gguf models.+ CPU - pip install llama-cpp-python+ CUDA - pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/<cuda-version> e.g. cu121 for CUDA 12.1+ METAL(Mac) - pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/metal+ others: https://pypi.org/project/llama-cpp-python/""",+ ) from exc++from typing import Any, Literal++from dsp.modules.lm import LM+++class LlamaCpp(LM):+ def __init__(+ self,+ model: str, # "llama" or the actual model name+ llama_model: Llama,+ model_type: Literal["chat", "text"] = None,+ **kwargs,+ ):+ super().__init__(model)++ default_model_type = "text"+ self.model_type = model_type if model_type else default_model_type+ self.provider = "llama"++ self.kwargs = {+ "temperature": 0.0,+ "max_tokens": 150,+ "top_p": 1,+ "frequency_penalty": 0,+ "presence_penalty": 0,+ **kwargs,+ } # TODO: add kwargs above for </s>++ self.loaded_model = llama_model+ self.history: list[dict[str, Any]] = []++ def basic_request(self, prompt: str, **kwargs):+ raw_kwargs = kwargs++ kwargs = {**self.kwargs, **kwargs}+ if "n" in kwargs:+ del kwargs["n"]++ if self.model_type == "chat":+ kwargs["messages"] = [{"role": "user", "content": prompt}]+ response = self.loaded_model.create_chat_completion(**kwargs)++ else:+ kwargs["prompt"] = prompt+ response = self.loaded_model.create_completion(**kwargs)++ history = {+ "prompt": prompt,+ "response": response,+ "kwargs": kwargs,+ "raw_kwargs": raw_kwargs,+ }+ self.history.append(history)+ return response++ def request(self, prompt: str, **kwargs):+ if "model_type" in kwargs:+ del kwargs["model_type"]++ return self.basic_request(prompt, **kwargs)++ def _get_choice_text(self, choice) -> str:+ if self.model_type == "chat":+ return choice["message"]["content"]+ return choice["text"]++ def __call__(+ self,+ prompt: str,+ only_completed: bool = True,+ return_sorted: bool = False,+ **kwargs,+ ) -> list[dict[str, Any]]:+ assert only_completed, "for now"+ assert return_sorted is False, "for now"++ response = self.request(prompt, **kwargs)+ choices = response["choices"]++ completed_choices = [c for c in choices if c["finish_reason"] != "length"]++ if only_completed and len(completed_choices):+ choices = completed_choices++ completions = [self._get_choice_text(c) for c in choices]++ return completionsdiff --git a/dsp/modules/lm.py b/dsp/modules/lm.pyindex 13ddfbffa..852ab44e0 100644--- a/dsp/modules/lm.py+++ b/dsp/modules/lm.py@@ -106,7 +106,7 @@ class LM(ABC):"tensorrt_llm",):text = choices- elif provider == "openai" or provider == "ollama":+ elif provider == "openai" or provider == "ollama" or provider == "llama":text = " " + self._get_choice_text(choices[0]).strip()elif provider == "groq":text = " " + choicesdiff --git a/dspy/__init__.py b/dspy/__init__.pyindex b56b82f37..08548461e 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -38,6 +38,7 @@ Anyscale = dsp.AnyscaleTogether = dsp.TogetherHFModel = dsp.HFModelOllamaLocal = dsp.OllamaLocal+LlamaCpp = dsp.LlamaCppBedrock = dsp.BedrockSagemaker = dsp.Sagemakerdiff --git a/pyproject.toml b/pyproject.tomlindex 9afb45e5e..26f4d2694 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -126,6 +126,7 @@ pgvector = { version = "^0.2.5", optional = true }structlog = "^24.1.0"llama-index = {version = "^0.10.30", optional = true}snowflake-snowpark-python = { version = "*",optional=true, python = ">=3.9,<3.12" }+llama-cpp-python = { version = "^0.2.82", optional = true }jinja2 = "^3.1.3"