Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Addressed Llamma Cpp Support PR Comments

Abdul Dridiac94c15cc8b4
diff --git a/docs/api/local_language_model_clients/LlamaCpp.md b/docs/api/local_language_model_clients/LlamaCpp.md
new file mode 100644
index 000000000..699505f70
--- /dev/null
+++ b/docs/api/local_language_model_clients/LlamaCpp.md
@@ -0,0 +1,42 @@
+# dspy.LlamaCpp
+
+## Prerequisites
+
+Install Llama Cpp Python by following the instructions provided in the [Llama Cpp Python repository](https://github.com/abetlen/llama-cpp-python).
+
+```shell
+pip install llama-cpp-python
+```
+
+alternatively, to install with CUDA support:
+
+```shell
+CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python
+```
+
+
+### Initializing the Llama model
+
+Initialize the model within your program with the desired parameters.
+
+```python
+from llama_cpp import Llama
+
+llm = Llama(
+ model_path="./sppo_finetuned_llama_3_8b.gguf",
+ n_gpu_layers=-1,
+ n_ctx=0,
+ verbose=False
+)
+```
+
+
+### Sending requests to the model
+
+After initializing the Llama model, you can interact with it using the `LlamaCpp` client.
+
+```python
+import dspy
+
+llamalm = dspy.LlamaCpp(model="llama", llama_model=llm, model_type="chat", temperature=0.4)
+```
\ No newline at end of file
diff --git a/dsp/modules/__init__.py b/dsp/modules/__init__.py
index 8428db4b2..c0aa98cd8 100644
--- a/dsp/modules/__init__.py
+++ b/dsp/modules/__init__.py
@@ -18,6 +18,7 @@ from .gpt3 import *
from .groq_client import *
from .hf import HFModel
from .hf_client import Anyscale, HFClientTGI, Together
+from .llama import *
from .mistral import *
from .ollama import *
from .premai import PremAI
diff --git a/dsp/modules/llama.py b/dsp/modules/llama.py
new file mode 100644
index 000000000..e1a920eac
--- /dev/null
+++ b/dsp/modules/llama.py
@@ -0,0 +1,99 @@
+try:
+ import llama_cpp
+ from llama_cpp import Llama
+except ImportError as exc:
+ raise ModuleNotFoundError(
+ """You need to install llama_cpp library to use gguf models.
+ CPU - pip install llama-cpp-python
+ CUDA - pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/<cuda-version> e.g. cu121 for CUDA 12.1
+ METAL(Mac) - pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/metal
+ others: https://pypi.org/project/llama-cpp-python/""",
+ ) from exc
+
+from typing import Any, Literal
+
+from dsp.modules.lm import LM
+
+
+class LlamaCpp(LM):
+ def __init__(
+ self,
+ model: str, # "llama" or the actual model name
+ llama_model: Llama,
+ model_type: Literal["chat", "text"] = None,
+ **kwargs,
+ ):
+ super().__init__(model)
+
+ default_model_type = "text"
+ self.model_type = model_type if model_type else default_model_type
+ self.provider = "llama"
+
+ self.kwargs = {
+ "temperature": 0.0,
+ "max_tokens": 150,
+ "top_p": 1,
+ "frequency_penalty": 0,
+ "presence_penalty": 0,
+ **kwargs,
+ } # TODO: add kwargs above for </s>
+
+ self.loaded_model = llama_model
+ self.history: list[dict[str, Any]] = []
+
+ def basic_request(self, prompt: str, **kwargs):
+ raw_kwargs = kwargs
+
+ kwargs = {**self.kwargs, **kwargs}
+ if "n" in kwargs:
+ del kwargs["n"]
+
+ if self.model_type == "chat":
+ kwargs["messages"] = [{"role": "user", "content": prompt}]
+ response = self.loaded_model.create_chat_completion(**kwargs)
+
+ else:
+ kwargs["prompt"] = prompt
+ response = self.loaded_model.create_completion(**kwargs)
+
+ history = {
+ "prompt": prompt,
+ "response": response,
+ "kwargs": kwargs,
+ "raw_kwargs": raw_kwargs,
+ }
+ self.history.append(history)
+ return response
+
+ def request(self, prompt: str, **kwargs):
+ if "model_type" in kwargs:
+ del kwargs["model_type"]
+
+ return self.basic_request(prompt, **kwargs)
+
+ def _get_choice_text(self, choice) -> str:
+ if self.model_type == "chat":
+ return choice["message"]["content"]
+ return choice["text"]
+
+ def __call__(
+ self,
+ prompt: str,
+ only_completed: bool = True,
+ return_sorted: bool = False,
+ **kwargs,
+ ) -> list[dict[str, Any]]:
+ assert only_completed, "for now"
+ assert return_sorted is False, "for now"
+
+ response = self.request(prompt, **kwargs)
+ choices = response["choices"]
+
+ completed_choices = [c for c in choices if c["finish_reason"] != "length"]
+
+ if only_completed and len(completed_choices):
+ choices = completed_choices
+
+ completions = [self._get_choice_text(c) for c in choices]
+
+ return completions
diff --git a/dsp/modules/lm.py b/dsp/modules/lm.py
index 13ddfbffa..852ab44e0 100644
--- a/dsp/modules/lm.py
+++ b/dsp/modules/lm.py
@@ -106,7 +106,7 @@ class LM(ABC):
"tensorrt_llm",
):
text = choices
- elif provider == "openai" or provider == "ollama":
+ elif provider == "openai" or provider == "ollama" or provider == "llama":
text = " " + self._get_choice_text(choices[0]).strip()
elif provider == "groq":
text = " " + choices
diff --git a/dspy/__init__.py b/dspy/__init__.py
index b56b82f37..08548461e 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -38,6 +38,7 @@ Anyscale = dsp.Anyscale
Together = dsp.Together
HFModel = dsp.HFModel
OllamaLocal = dsp.OllamaLocal
+LlamaCpp = dsp.LlamaCpp
Bedrock = dsp.Bedrock
Sagemaker = dsp.Sagemaker
diff --git a/pyproject.toml b/pyproject.toml
index 9afb45e5e..26f4d2694 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -126,6 +126,7 @@ pgvector = { version = "^0.2.5", optional = true }
structlog = "^24.1.0"
llama-index = {version = "^0.10.30", optional = true}
snowflake-snowpark-python = { version = "*",optional=true, python = ">=3.9,<3.12" }
+llama-cpp-python = { version = "^0.2.82", optional = true }
jinja2 = "^3.1.3"