DSPy: The framework for programming—not prompting—language models
Python
Adding local model integration with Ollama
b-akshay9dde4bda29f3
diff --git a/dsp/modules/lm.py b/dsp/modules/lm.pyindex 468c350ab..490aa6df8 100644--- a/dsp/modules/lm.py+++ b/dsp/modules/lm.py@@ -69,7 +69,7 @@ class LM(ABC):text = ""if provider == "cohere":text = choices[0].text- elif provider == "openai":+ elif provider == "openai" or provider == "ollama":text = ' ' + self._get_choice_text(choices[0]).strip()else:text = choices[0]["text"]diff --git a/dsp/modules/ollama.py b/dsp/modules/ollama.pynew file mode 100644index 000000000..7e35d0b61--- /dev/null+++ b/dsp/modules/ollama.py@@ -0,0 +1,168 @@+from dsp.modules.lm import LM+from typing import Any, Literal, Optional++import os, multiprocessing, datetime, hashlib+import requests++import json++def post_request_metadata(model_name, prompt):+ """Creates a serialized request object for the Ollama API."""+ timestamp = datetime.datetime.now().timestamp()+ id_string = str(timestamp) + model_name + prompt+ hashlib.sha1().update(id_string.encode("utf-8"))+ id_hash = hashlib.sha1().hexdigest()+ return {+ "id": f"chatcmpl-{id_hash}",+ "object": "chat.completion",+ "created": int(timestamp),+ "model": model_name+ }+++class OllamaLocal(LM):+ """Wrapper around a locally hosted Ollama model (API: https://github.com/jmorganca/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values and https://github.com/jmorganca/ollama/blob/main/docs/api.md#generate-a-completion).+ Returns dictionary info in the OpenAI API style (https://platform.openai.com/docs/api-reference/chat/object).++ Args:+ model (str, optional): Name of Ollama model. Defaults to "llama2".+ model_type (Literal["chat", "text"], optional): The type of model that was specified. Mainly to decide the optimal prompting strategy. Defaults to "text".+ **kwargs: Additional arguments to pass to the API.+ """++ def __init__(+ self,+ model: str = "llama2",+ model_type: Literal["chat", "text"] = None,+ **kwargs,+ ):+ super().__init__(model)+ self.provider = "ollama"++ self.base_url = "http://localhost:11434" # where the model is hosted++ default_model_type = "text"+ self.model_type = model_type if model_type else default_model_type++ self.model_name = model+ self.num_cores = multiprocessing.cpu_count()-1+ self.timeout_duration = 15.0 # Seconds to wait for http requests to Ollama server++ self.kwargs = {+ "temperature": 0.0,+ "max_tokens": 150,+ "top_p": 1,+ "top_k": 20,+ "frequency_penalty": 0,+ "presence_penalty": 0,+ "n": 1,+ "num_ctx": 1024,+ **kwargs,+ }+ self.kwargs["num_predict"] = self.kwargs["max_tokens"] # Ollama uses num_predict instead of max_tokens++ self.history: list[dict[str, Any]] = []++ self.version = ''+ if 'version' in kwargs:+ self.version = kwargs['version']++ def basic_request(self, prompt: str, **kwargs):++ raw_kwargs = kwargs++ kwargs = {**self.kwargs, **kwargs}++ request_info = post_request_metadata(self.model_name, prompt)+ request_info["choices"] = []+ settings_dict = {+ "model": self.model_name,+ "options": {k: v for k, v in kwargs.items() if k not in ["n", "max_tokens"]},+ "stream": False,+ }+ if self.model_type == "chat":+ settings_dict["messages"] = [{"role": "user", "content": prompt}]+ else:+ settings_dict["prompt"] = prompt++ urlstr = f"{self.base_url}/api/chat" if self.model_type == "chat" else f"{self.base_url}/api/generate"+ tot_eval_tokens = 0+ for i in range(kwargs["n"]):+ response = requests.post(urlstr, json=settings_dict)++ # Check if the request was successful (HTTP status code 200)+ if response.status_code != 200:+ # If the request was not successful, print an error message+ print(f"Error: CODE {response.status_code} - {response.text}")++ response_json = response.json()++ text = response_json.get("message").get("content") if self.model_type == "chat" else response_json.get("response")+ request_info["choices"].append({+ "index": i,+ "message": { "role": "assistant", "content": ''.join(text), },+ "finish_reason": "stop"+ })+ tot_eval_tokens += response_json.get("eval_count")+ request_info["additional_kwargs"] = {k: v for k, v in response_json.items() if k not in ["response"]}++ print('RESPONSE JSON', response_json)+ request_info["usage"] = {+ "prompt_tokens": response_json.get("prompt_eval_count"),+ "completion_tokens": tot_eval_tokens,+ "total_tokens": response_json.get("prompt_eval_count") + tot_eval_tokens+ }++ history = {+ "prompt": prompt,+ "response": request_info,+ "kwargs": kwargs,+ "raw_kwargs": raw_kwargs,+ }+ self.history.append(history)++ return request_info++ def request(self, prompt: str, **kwargs):+ """Wrapper for requesting completions from the Ollama model."""+ if "model_type" in kwargs:+ del kwargs["model_type"]++ return self.basic_request(prompt, **kwargs)++ def _get_choice_text(self, choice: dict[str, Any]) -> str:+ return choice["message"]["content"]++ def __call__(+ self,+ prompt: str,+ only_completed: bool = True,+ return_sorted: bool = False,+ **kwargs,+ ) -> list[dict[str, Any]]:+ """Retrieves completions from Ollama.++ Args:+ prompt (str): prompt to send to Ollama+ only_completed (bool, optional): return only completed responses and ignores completion due to length. Defaults to True.+ return_sorted (bool, optional): sort the completion choices using the returned probabilities. Defaults to False.++ Returns:+ list[dict[str, Any]]: list of completion choices+ """++ assert only_completed, "for now"+ assert return_sorted is False, "for now"++ response = self.request(prompt, **kwargs)++ choices = response["choices"]++ completed_choices = [c for c in choices if c["finish_reason"] != "length"]++ if only_completed and len(completed_choices):+ choices = completed_choices+ print(choices)+ completions = [self._get_choice_text(c) for c in choices]++ return completions\ No newline at end of file