Skip to content

Devika is the first open-source implementation of an Agentic Software Engineer. Initially started as an open-source alternative to Devin.

Python

Feature: Added LlamaCpp support for Local inference

rajakumar050320003a608de55bbd
diff --git a/config.toml b/config.toml
index 8e45dbc..41d37ec 100644
--- a/config.toml
+++ b/config.toml
@@ -5,6 +5,7 @@ PDFS_DIR = "pdfs"
PROJECTS_DIR = "projects"
LOGS_DIR = "logs"
REPOS_DIR = "repos"
+MODEL_PATH = "<YOUR_MODEL_PATH>"
[API_KEYS]
BING = "<YOUR_BING_API_KEY>"
@@ -14,3 +15,8 @@ OPENAI = "<YOUR_OPENAI_API_KEY>"
[API_ENDPOINTS]
BING = "https://api.bing.microsoft.com/v7.0/search"
+
+#Only parameters are required if you are using LlamaCpp model
+[PARAMETERS]
+GPU_LAYERS = "<YOUR_GPU_LAYERS>"
+CONTEXT_SIZE = "<YOUR_CONEXT_SIZE>"
diff --git a/requirements.txt b/requirements.txt
index 53496f4..649418e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -20,4 +20,5 @@ keybert
GitPython
netlify-py
Markdown
-xhtml2pdf
\ No newline at end of file
+xhtml2pdf
+llama-cpp-python
\ No newline at end of file
diff --git a/src/browser/search.py b/src/browser/search.py
index daa22d8..7c83d52 100644
--- a/src/browser/search.py
+++ b/src/browser/search.py
@@ -21,5 +21,4 @@ class BingSearch:
return err
def get_first_link(self):
- return self.query_result["webPages"]["value"][0]["url"]
-
+ return self.query_result["webPages"]["value"][0]["url"]
\ No newline at end of file
diff --git a/src/config.py b/src/config.py
index 4f0d06b..3b33721 100644
--- a/src/config.py
+++ b/src/config.py
@@ -39,6 +39,15 @@ class Config:
def get_repos_dir(self):
return self.config["STORAGE"]["REPOS_DIR"]
+
+ def get_model_path(self):
+ return self.config["STORAGE"]["MODEL_PATH"]
+
+ def get_gpu_layers(self):
+ return self.config["PARAMETERS"]["GPU_LAYERS"]
+
+ def get_context_size(self):
+ return self.config["PARAMETERS"]["CONTEXT_SIZE"]
def set_bing_api_key(self, key):
self.config["API_KEYS"]["BING"] = key
@@ -84,6 +93,19 @@ class Config:
self.config["STORAGE"]["REPOS_DIR"] = dir
self.save_config()
+ def set_model_path(self, model_path):
+ self.config["STORAGE"]["MODEL_PATH"] = model_path
+ self.save_config()
+
+ def set_gpu_layers(self, gpu_layers):
+ self.config["PARAMETERS"]["GPU_LAYERS"] = gpu_layers
+ self.save_config()
+
+ def set_context_size(self, context_size):
+ self.config["PARAMETERS"]["CONTEXT_SIZE"] = context_size
+ self.save_config()
+
def save_config(self):
with open("config.toml", "w") as f:
toml.dump(self.config, f)
+
diff --git a/src/llm/llama_cpp_client.py b/src/llm/llama_cpp_client.py
new file mode 100644
index 0000000..8d3399a
--- /dev/null
+++ b/src/llm/llama_cpp_client.py
@@ -0,0 +1,29 @@
+import llama_cpp
+from src.logger import Logger
+from src.config import Config
+class LlamaCpp:
+ _instance = None
+
+ def __new__(cls):
+ if not cls._instance:
+ cls._instance = super().__new__(cls)
+ cls._instance.initialized = False
+ return cls._instance
+
+ def __init__(self):
+ if not self.initialized:
+ config = Config()
+ model_path = config.get_model_path()
+ gpu_layers = int(config.get_gpu_layers())
+ context_size = int(config.get_context_size())
+ self.model = llama_cpp.Llama(model_path=model_path, n_gpu_layers=gpu_layers, n_ctx=context_size)
+ self.initialized = True
+
+ def inference(self, prompt):
+ response = self.model.create_completion(prompt=prompt, max_tokens=2048)
+ return response["choices"][0]["text"]
+
+
+
+
+
diff --git a/src/llm/llm.py b/src/llm/llm.py
index 4080b5e..d40ec35 100644
--- a/src/llm/llm.py
+++ b/src/llm/llm.py
@@ -3,6 +3,7 @@ from enum import Enum
from .ollama_client import Ollama
from .claude_client import Claude
from .openai_client import OpenAI
+from .llama_cpp_client import LlamaCpp
import tiktoken
@@ -15,6 +16,7 @@ class Model(Enum):
CLAUDE_3_HAIKU = ("Claude 3 Haiku", "claude-3-haiku-20240307")
GPT_4_TURBO = ("GPT-4 Turbo", "gpt-4-0125-preview")
GPT_3_5 = ("GPT-3.5", "gpt-3.5-turbo-0125")
+ LLAMA_CPP = ("LLAMA_CPP", "LLAMA_CPP_MODEL")
OLLAMA_MODELS = [
(
model["name"].split(":")[0],
@@ -56,6 +58,8 @@ class LLM:
response = Claude().inference(self.model_id, prompt).strip()
elif "GPT" in str(model):
response = OpenAI().inference(self.model_id, prompt).strip()
+ elif "LLAMA_CPP" in str(model):
+ response = LlamaCpp().inference(prompt).strip()
else:
raise ValueError(f"Model {model} not supported")