Devika is the first open-source implementation of an Agentic Software Engineer. Initially started as an open-source alternative to Devin.
Python
Feature: Added LlamaCpp support for Local inference
rajakumar050320003a608de55bbd
diff --git a/config.toml b/config.tomlindex 8e45dbc..41d37ec 100644--- a/config.toml+++ b/config.toml@@ -5,6 +5,7 @@ PDFS_DIR = "pdfs"PROJECTS_DIR = "projects"LOGS_DIR = "logs"REPOS_DIR = "repos"+MODEL_PATH = "<YOUR_MODEL_PATH>"[API_KEYS]BING = "<YOUR_BING_API_KEY>"@@ -14,3 +15,8 @@ OPENAI = "<YOUR_OPENAI_API_KEY>"[API_ENDPOINTS]BING = "https://api.bing.microsoft.com/v7.0/search"++#Only parameters are required if you are using LlamaCpp model+[PARAMETERS]+GPU_LAYERS = "<YOUR_GPU_LAYERS>"+CONTEXT_SIZE = "<YOUR_CONEXT_SIZE>"diff --git a/requirements.txt b/requirements.txtindex 53496f4..649418e 100644--- a/requirements.txt+++ b/requirements.txt@@ -20,4 +20,5 @@ keybertGitPythonnetlify-pyMarkdown-xhtml2pdf\ No newline at end of file+xhtml2pdf+llama-cpp-python\ No newline at end of filediff --git a/src/browser/search.py b/src/browser/search.pyindex daa22d8..7c83d52 100644--- a/src/browser/search.py+++ b/src/browser/search.py@@ -21,5 +21,4 @@ class BingSearch:return errdef get_first_link(self):- return self.query_result["webPages"]["value"][0]["url"]-+ return self.query_result["webPages"]["value"][0]["url"]\ No newline at end of filediff --git a/src/config.py b/src/config.pyindex 4f0d06b..3b33721 100644--- a/src/config.py+++ b/src/config.py@@ -39,6 +39,15 @@ class Config:def get_repos_dir(self):return self.config["STORAGE"]["REPOS_DIR"]++ def get_model_path(self):+ return self.config["STORAGE"]["MODEL_PATH"]++ def get_gpu_layers(self):+ return self.config["PARAMETERS"]["GPU_LAYERS"]++ def get_context_size(self):+ return self.config["PARAMETERS"]["CONTEXT_SIZE"]def set_bing_api_key(self, key):self.config["API_KEYS"]["BING"] = key@@ -84,6 +93,19 @@ class Config:self.config["STORAGE"]["REPOS_DIR"] = dirself.save_config()+ def set_model_path(self, model_path):+ self.config["STORAGE"]["MODEL_PATH"] = model_path+ self.save_config()++ def set_gpu_layers(self, gpu_layers):+ self.config["PARAMETERS"]["GPU_LAYERS"] = gpu_layers+ self.save_config()++ def set_context_size(self, context_size):+ self.config["PARAMETERS"]["CONTEXT_SIZE"] = context_size+ self.save_config()+def save_config(self):with open("config.toml", "w") as f:toml.dump(self.config, f)+diff --git a/src/llm/llama_cpp_client.py b/src/llm/llama_cpp_client.pynew file mode 100644index 0000000..8d3399a--- /dev/null+++ b/src/llm/llama_cpp_client.py@@ -0,0 +1,29 @@+import llama_cpp+from src.logger import Logger+from src.config import Config+class LlamaCpp:+ _instance = None++ def __new__(cls):+ if not cls._instance:+ cls._instance = super().__new__(cls)+ cls._instance.initialized = False+ return cls._instance++ def __init__(self):+ if not self.initialized:+ config = Config()+ model_path = config.get_model_path()+ gpu_layers = int(config.get_gpu_layers())+ context_size = int(config.get_context_size())+ self.model = llama_cpp.Llama(model_path=model_path, n_gpu_layers=gpu_layers, n_ctx=context_size)+ self.initialized = True++ def inference(self, prompt):+ response = self.model.create_completion(prompt=prompt, max_tokens=2048)+ return response["choices"][0]["text"]+++++diff --git a/src/llm/llm.py b/src/llm/llm.pyindex 4080b5e..d40ec35 100644--- a/src/llm/llm.py+++ b/src/llm/llm.py@@ -3,6 +3,7 @@ from enum import Enumfrom .ollama_client import Ollamafrom .claude_client import Claudefrom .openai_client import OpenAI+from .llama_cpp_client import LlamaCppimport tiktoken@@ -15,6 +16,7 @@ class Model(Enum):CLAUDE_3_HAIKU = ("Claude 3 Haiku", "claude-3-haiku-20240307")GPT_4_TURBO = ("GPT-4 Turbo", "gpt-4-0125-preview")GPT_3_5 = ("GPT-3.5", "gpt-3.5-turbo-0125")+ LLAMA_CPP = ("LLAMA_CPP", "LLAMA_CPP_MODEL")OLLAMA_MODELS = [(model["name"].split(":")[0],@@ -56,6 +58,8 @@ class LLM:response = Claude().inference(self.model_id, prompt).strip()elif "GPT" in str(model):response = OpenAI().inference(self.model_id, prompt).strip()+ elif "LLAMA_CPP" in str(model):+ response = LlamaCpp().inference(prompt).strip()else:raise ValueError(f"Model {model} not supported")