Devika is the first open-source implementation of an Agentic Software Engineer. Initially started as an open-source alternative to Devin.
Python
fix: duckduckgo update: UI and backend
ARajgor7cd567b157bd
diff --git a/devika.py b/devika.pyindex 32b4ea4..5fe0930 100644--- a/devika.py+++ b/devika.py@@ -44,8 +44,6 @@ AgentState = AgentState()config = Config()logger = Logger()-logger.mode = "on"-# initial socket@socketio.on('socket_connect')@@ -59,7 +57,7 @@ def test_connect(data):def data():project = manager.get_project_list()models = LLM().list_models()- search_engines = ["Bing", "Google"]+ search_engines = ["Bing", "Google", "DuckDuckGo"]return jsonify({"projects": project, "models": models, "search_engines": search_engines})@@ -78,7 +76,7 @@ def handle_message(data):message = data.get('message')base_model = data.get('base_model')project_name = data.get('project_name')- search_engine = data.get('search_engine')+ search_engine = data.get('search_engine').lower()agent = Agent(base_model=base_model, search_engine=search_engine)@@ -89,12 +87,11 @@ def handle_message(data):manager.add_message_to_project(project_name, new_message)if AgentState.is_agent_completed(project_name):- agent.subsequent_execute(message, project_name)thread = Thread(target=lambda: agent.subsequent_execute(message, project_name))thread.start()if action == 'execute_agent':- thread = Thread(target=lambda: agent.execute(message, project_name))+ thread = Thread(target=lambda: agent.execute(message, project_name, search_engine))thread.start()@@ -184,6 +181,7 @@ def real_time_logs():@route_logger(logger)def set_settings():data = request.json+ print("Data: ", data)config.config.update(data)config.save_config()return jsonify({"message": "Settings updated"})diff --git a/requirements.txt b/requirements.txtindex 5d456af..ee38af1 100644--- a/requirements.txt+++ b/requirements.txt@@ -27,3 +27,4 @@ Flask-SocketIOeventletgroqduckduckgo-search+orjson\ No newline at end of filediff --git a/sample.config.toml b/sample.config.tomlindex 5a9ab51..03de6c3 100644--- a/sample.config.toml+++ b/sample.config.toml@@ -5,18 +5,18 @@ PDFS_DIR = "data/pdfs"PROJECTS_DIR = "data/projects"LOGS_DIR = "data/logs"REPOS_DIR = "data/repos"+WEB_SEARCH = "ddgs"[API_KEYS]NETLIFY = "<YOUR_NETLIFY_API_KEY>"BING = "<YOUR_BING_API_KEY>"GOOGLE = "<YOUR_GOOGLE_SEARCH_API_KEY>"-GOOGLE_CX = "<YOUR_GOOGLE_SEARCH_ENGINE_ID>"+GOOGLE_SEARCH_ENGINE_ID = "<YOUR_GOOGLE_SEARCH_ENGINE_ID>"CLAUDE = "<YOUR_CLAUDE_API_KEY>"OPENAI = "<YOUR_OPENAI_API_KEY>"GEMINI = "<YOUR_GEMINI_API_KEY>"MISTRAL = "<YOUR_MISTRAL_API_KEY>"GROQ = "<YOUR_GROQ_API_KEY>"-GEMINI = "<YOUR_GEMINI_API_KEY>"[API_ENDPOINTS]BING = "https://api.bing.microsoft.com/v7.0/search"diff --git a/src/agents/agent.py b/src/agents/agent.pyindex 251a0f4..92fa935 100644--- a/src/agents/agent.py+++ b/src/agents/agent.py@@ -11,15 +11,13 @@ from .patcher import Patcherfrom .reporter import Reporterfrom .decision import Decision-from src.config import Configfrom src.project import ProjectManagerfrom src.state import AgentState-from src.socket_instance import emit_agentfrom src.logger import Loggerfrom src.bert.sentence import SentenceBertfrom src.memory import KnowledgeBase-from src.browser.search import BingSearch,DuckDuckGoSearch,GoogleSearch+from src.browser.search import BingSearch, GoogleSearch, DuckDuckGoSearchfrom src.browser import Browserfrom src.browser import start_interactionfrom src.filesystem import ReadCode@@ -65,59 +63,43 @@ class Agent:self.engine = search_engineself.tokenizer = tiktoken.get_encoding("cl100k_base")- def search_queries(self, queries: list, project_name: str, requested_web_search: str) -> dict:+ def search_queries(self, queries: list, project_name: str, engine: str) -> dict:results = {}knowledge_base = KnowledgeBase()-web_search = None- web_search_type = Config().get_web_search()- if requested_web_search:- web_search_type = requested_web_search- if web_search_type == "bing":+ if engine == "bing":web_search = BingSearch()- elif web_search_type == "google":+ elif engine == "google":web_search = GoogleSearch()- elif web_search_type == "ddgs":- web_search = DuckDuckGoSearch()else:web_search = DuckDuckGoSearch()-- self.logger.info(web_search_type)- browser = Browser()++ self.logger.info(f"Search : {engine}")for query in queries:+ browser = Browser()query = query.strip().lower()- """ Check if the knowledge base already has the query learned """# knowledge = knowledge_base.get_knowledge(tag=query)# if knowledge:# results[query] = knowledge# continue- """ Search for the query and get the first link """web_search.search(query)+link = web_search.get_first_link()- print("Link :: ", link)+ print("\nLink :: ", link, '\n')- """ Browse to the link and take a screenshot, then extract the text """browser.go_to(link)browser.screenshot(project_name)- """- Formatter Agent is invoked to format and learn from the contents- """- results[query] = self.formatter.execute(- browser.extract_text(),- project_name- )-- """- Add the newly acquired data to the knowledge base- """- # knowledge_base.add_knowledge(tag=query, contents=results[query])+ results[query] = self.formatter.execute(browser.extract_text(), project_name)+ # browser.close()+ self.logger.info(f"got the search results for : {query}")+ # knowledge_base.add_knowledge(tag=query, contents=results[query])return resultsdef update_contextual_keywords(self, sentence: str):@@ -132,7 +114,7 @@ class Agent:def make_decision(self, prompt: str, project_name: str) -> str:decision = self.decision.execute(prompt, project_name)-+for item in decision:function = item["function"]args = item["args"]@@ -170,7 +152,7 @@ class Agent:# Call the planner, researcher, coder agents in sequenceplan = self.planner.execute(user_prompt, project_name)planner_response = self.planner.parse_response(plan)-+research = self.researcher.execute(plan, self.collected_context_keywords, project_name)search_results = self.search_queries(research["queries"], project_name)@@ -198,9 +180,7 @@ class Agent:self.project_manager.add_message_from_devika(project_name, response)- print("=====" * 10)- print(action)- print("=====" * 10)+ print("\naction :: ", action, '\n')if action == "answer":response = self.answer.execute(@@ -209,10 +189,9 @@ class Agent:project_name=project_name)self.project_manager.add_message_from_devika(project_name, response)- elif action == "run":+ elif action == "run":project_path = self.project_manager.get_project_path(project_name)-self.runner.execute(conversation=conversation,code_markdown=code_markdown,@@ -220,6 +199,7 @@ class Agent:project_path=project_path,project_name=project_name)+elif action == "deploy":deploy_metadata = Netlify().deploy(project_name)deploy_url = deploy_metadata["deploy_url"]@@ -231,6 +211,7 @@ class Agent:response = json.dumps(response, indent=4)self.project_manager.add_message_from_devika(project_name, response)+elif action == "feature":code = self.feature.execute(conversation=conversation,@@ -238,10 +219,9 @@ class Agent:system_os=os_system,project_name=project_name)- print(code)- print("=====" * 10)-+ print("\nfeature code :: ", code, '\n')self.feature.save_code_to_project(code, project_name)+elif action == "bug":code = self.patcher.execute(conversation=conversation,@@ -251,10 +231,9 @@ class Agent:system_os=os_system,project_name=project_name)- print(code)- print("=====" * 10)-+ print("\nbug code :: ", code, '\n')self.patcher.save_code_to_project(code, project_name)+elif action == "report":markdown = self.reporter.execute(conversation, code_markdown, project_name)@@ -265,16 +244,16 @@ class Agent:project_name_space_url)response = f"I have generated the PDF document. You can download it from here: {pdf_download_url}"- Browser().go_to(pdf_download_url)- Browser().screenshot(project_name)+ browser = Browser()+ browser.go_to(pdf_download_url)+ browser.screenshot(project_name)self.project_manager.add_message_from_devika(project_name, response)self.agent_state.set_agent_active(project_name, False)self.agent_state.set_agent_completed(project_name, True)-- def execute(self, prompt: str, project_name_from_user: str = None, web_search: str = None) -> str:+ def execute(self, prompt: str, project_name_from_user: str = None, engine: str = None) -> str:"""Agentic flow of execution"""@@ -282,8 +261,7 @@ class Agent:self.project_manager.add_message_from_user(project_name_from_user, prompt)plan = self.planner.execute(prompt, project_name_from_user)- print("\nplan :: ", plan)- print("=====" * 10)+ print("\nplan :: ", plan, '\n')planner_response = self.planner.parse_response(plan)project_name = planner_response["project"]@@ -298,28 +276,25 @@ class Agent:project_name = planner_response["project"]self.project_manager.create_project(project_name)self.project_manager.add_message_from_user(project_name, prompt)-+self.agent_state.set_agent_active(project_name, True)- print("\nplans:: ", plans)self.project_manager.add_message_from_devika(project_name, reply)self.project_manager.add_message_from_devika(project_name, json.dumps(plans, indent=4))# self.project_manager.add_message_from_devika(project_name, f"In summary: {summary}")self.update_contextual_keywords(focus)- print("\ncontext_keywords :: ", self.collected_context_keywords)+ print("\ncontext_keywords :: ", self.collected_context_keywords, '\n')internal_monologue = self.internal_monologue.execute(current_prompt=plan, project_name=project_name)- print("\ninternal_monologue :: ", internal_monologue)- print("=====" * 10)+ print("\ninternal_monologue :: ", internal_monologue, '\n')new_state = self.agent_state.new_state()new_state["internal_monologue"] = internal_monologueself.agent_state.add_to_current_state(project_name, new_state)research = self.researcher.execute(plan, self.collected_context_keywords, project_name=project_name)- print("\nresearch :: ", research)- print("=====" * 10)+ print("\nresearch :: ", research, '\n')queries = research["queries"]queries_combined = ", ".join(queries)@@ -332,7 +307,8 @@ class Agent:f"\n If I need anything, I will make sure to ask you.")if not queries and len(queries) == 0:- self.project_manager.add_message_from_devika(project_name, "I think I can proceed without searching the web.")+ self.project_manager.add_message_from_devika(project_name,+ "I think I can proceed without searching the web.")ask_user_prompt = "Nothing from the user."@@ -353,18 +329,12 @@ class Agent:got_user_query = Trueself.project_manager.add_message_from_devika(project_name, "Thanks! 🙌")time.sleep(5)-- AgentState().set_agent_active(project_name, True)-- search_results = self.search_queries(queries, project_name, web_search)self.agent_state.set_agent_active(project_name, True)if queries and len(queries) > 0:- search_results = self.search_queries(queries, project_name)- print("search_results :: ", search_results)- print(json.dumps(search_results, indent=4))- print("=====" * 10)+ search_results = self.search_queries(queries, project_name, engine)+else:search_results = {}@@ -374,12 +344,13 @@ class Agent:search_results=search_results,project_name=project_name)- print(code)- print("=====" * 10)+ print("\ncode :: ", code, '\n')self.coder.save_code_to_project(code, project_name)self.agent_state.set_agent_active(project_name, False)self.agent_state.set_agent_completed(project_name, True)- self.project_manager.add_message_from_devika(project_name, "I have completed the coding task. You can now run the project.")-+ self.project_manager.add_message_from_devika(project_name,+ "I have completed the my task. 🚀 \n"+ "if you would like me to do anything else, please let me know. \n"+ )diff --git a/src/agents/researcher/prompt.jinja2 b/src/agents/researcher/prompt.jinja2index 4c05783..19ff718 100644--- a/src/agents/researcher/prompt.jinja2+++ b/src/agents/researcher/prompt.jinja2@@ -1,7 +1,7 @@For the provided step-by-step plan, write all the necessary search queries to gather information from the web that the base model doesn't already know.Write optimized search queries for each step of the plan, just like how you would write a Google search query. Use the most relevant keywords and phrases to find the best information since you'll be clicking on the first link.-if you feel that you already know how to perform the steps and you have the knowledge to do so then don't need to search for anything, leave queries field empty.+also only ask for information if you think it's necessary, otherwise leave ask_user field empty.Step-by-Step Plan:diff --git a/src/browser/browser.py b/src/browser/browser.pyindex ab9c504..11f5645 100644--- a/src/browser/browser.py+++ b/src/browser/browser.py@@ -12,7 +12,7 @@ class Browser:def __init__(self):self.playwright = sync_playwright().start()chromium = self.playwright.chromium- self.browser = chromium.launch(headless=True)+ self.browser = chromium.launch()self.page = self.browser.new_page()def new_page(self):@@ -20,7 +20,8 @@ class Browser:def go_to(self, url):try:- self.page.goto(url, wait_until="load", timeout=30000)+ self.page.goto(url, timeout=20000)+except TimeoutError as e:print(f"TimeoutError: {e} when trying to navigate to {url}")return Falsediff --git a/src/browser/search.py b/src/browser/search.pyindex 8038e40..6b140ff 100644--- a/src/browser/search.py+++ b/src/browser/search.py@@ -1,6 +1,12 @@+import threading+import requestsfrom src.config import Config-from duckduckgo_search import DDGS++import re+from urllib.parse import unquote+from html import unescape+import orjsonclass BingSearch:@@ -19,12 +25,12 @@ class BingSearch:response.raise_for_status()self.query_result = response.json()return self.query_result- except Exception as err:- return err+ except Exception as error:+ return errordef get_first_link(self):return self.query_result["webPages"]["value"][0]["url"]-+class GoogleSearch:def __init__(self):@@ -33,7 +39,7 @@ class GoogleSearch:self.google_search_engine_ID = self.config.get_google_search_engine_id()self.google_search_api_endpoint = self.config.get_google_search_api_endpoint()self.query_result = None-+def search(self, query):params = {"key": self.google_search_api_key,@@ -42,7 +48,7 @@ class GoogleSearch:}try:print("Searching in Google...")- response = requests.get(self.google_api_endpoint, params=params)+ response = requests.get(self.google_search_api_endpoint, params=params)# response.raise_for_status()self.query_result = response.json()except Exception as error:@@ -53,18 +59,106 @@ class GoogleSearch:if 'items' in self.query_result:item = self.query_result['items'][0]['link']return item-+++# class DuckDuckGoSearch:+# def __init__(self):+# self.query_result = None+#+# def search(self, query):+# from duckduckgo_search import DDGS+# try:+# self.query_result = DDGS().text(query, max_results=5, region="us")+# print(self.query_result)+#+# except Exception as err:+# print(err)+#+# def get_first_link(self):+# if self.query_result:+# return self.query_result[0]["href"]+# else:+# return None+#+class DuckDuckGoSearch:+ """DuckDuckGo search engine class.+ methods are inherited from the duckduckgo_search package.+ do not change the methods.++ currently, the package is not working with our current setup.+ """def __init__(self):+ from curl_cffi import requests as curl_requestsself.query_result = None-- def search(self, query):+ self.asession = curl_requests.Session(impersonate="chrome", allow_redirects=False)+ self.asession.headers["Referer"] = "https://duckduckgo.com/"++ def _get_url(self, method, url, data):try:- self.query_result = DDGS().text(query, max_results=5)- return self.query_result- except Exception as err:- print(err)+ resp = self.asession.request(method, url, data=data)+ except Exception as error:+ if "timeout" in str(error).lower():+ raise TimeoutError("Duckduckgo timed out error")+ if resp.status_code == 200:+ return resp.content+ if resp.status_code == (202, 301, 403):+ raise Exception(f"Error: {resp.status_code} rate limit error")++ def duck(self, query):+ resp = self._get_url("POST", "https://duckduckgo.com/", data={"q": query})+ vqd = self.extract_vqd(resp)++ params = {"q": query, "kl": 'en-us', "p": "1", "s": "0", "df": "", "vqd": vqd, "ex": ""}+ resp = self._get_url("GET", "https://links.duckduckgo.com/d.js", params)+ page_data = self.text_extract_json(resp)++ results = []+ for row in page_data:+ href = row.get("u")+ if href and href != f"http://www.google.com/search?q={query}":+ body = self.normalize(row["a"])+ if body:+ result = {+ "title": self.normalize(row["t"]),+ "href": self.normalize_url(href),+ "body": self.normalize(row["a"]),+ }+ results.append(result)++ self.query_result = results++ def search(self, query):+ self.duck(query)def get_first_link(self):return self.query_result[0]["href"]++ @staticmethod+ def extract_vqd(html_bytes: bytes) -> str:+ patterns = [(b'vqd="', 5, b'"'), (b"vqd=", 4, b"&"), (b"vqd='", 5, b"'")]+ for start_pattern, offset, end_pattern in patterns:+ try:+ start = html_bytes.index(start_pattern) + offset+ end = html_bytes.index(end_pattern, start)+ return html_bytes[start:end].decode()+ except ValueError:+ continue++ @staticmethod+ def text_extract_json(html_bytes):+ try:+ start = html_bytes.index(b"DDG.pageLayout.load('d',") + 24+ end = html_bytes.index(b");DDG.duckbar.load(", start)+ return orjson.loads(html_bytes[start:end])+ except Exception as ex:+ print(f"Error extracting JSON: {type(ex).__name__}: {ex}")++ @staticmethod+ def normalize_url(url: str) -> str:+ return unquote(url.replace(" ", "+")) if url else ""++ @staticmethod+ def normalize(raw_html: str) -> str:+ return unescape(re.sub("<.*?>", "", raw_html)) if raw_html else ""diff --git a/src/config.py b/src/config.pyindex 90f6b14..f72547e 100644--- a/src/config.py+++ b/src/config.py@@ -14,15 +14,6 @@ class Config:def get_config(self):return self.config- def get_google_api_endpoint(self):- return self.config["API_ENDPOINTS"]["GOOGLE"]-- def get_google_api_key(self):- return self.config["API_KEYS"]["GOOGLE"]-- def get_google_cx(self):- return self.config["API_KEYS"]["GOOGLE_CX"]-def get_bing_api_endpoint(self):return self.config["API_ENDPOINTS"]["BING"]@@ -53,6 +44,9 @@ class Config:def get_mistral_api_key(self):return self.config["API_KEYS"]["MISTRAL"]+ def get_groq_api_key(self):+ return self.config["API_KEYS"]["GROQ"]+def get_netlify_api_key(self):return self.config["API_KEYS"]["NETLIFY"]@@ -75,7 +69,7 @@ class Config:return self.config["STORAGE"]["REPOS_DIR"]def get_web_search(self):- return environ.get("WEB_SEARCH", self.config["STORAGE"]["WEB_SEARCH"])+ return self.config["STORAGE"]["WEB_SEARCH"]def get_logging_rest_api(self):return self.config["LOGGING"]["LOG_REST_API"] == "true"@@ -115,10 +109,18 @@ class Config:self.config["API_KEYS"]["OPENAI"] = keyself.save_config()- def set_openai_api_key(self, key):+ def set_gemini_api_key(self, key):self.config["API_KEYS"]["GEMINI"] = keyself.save_config()+ def set_mistral_api_key(self, key):+ self.config["API_KEYS"]["MISTRAL"] = key+ self.save_config()++ def set_groq_api_key(self, key):+ self.config["API_KEYS"]["GROQ"] = key+ self.save_config()+def set_netlify_api_key(self, key):self.config["API_KEYS"]["NETLIFY"] = keyself.save_config()diff --git a/src/init.py b/src/init.pyindex e0f895a..8a2207e 100644--- a/src/init.py+++ b/src/init.py@@ -1,22 +1,7 @@import os-import sys-from src.config import Configfrom src.logger import Logger-def init_cmd():- config = Config()- logger = Logger()- command_line_args = sys.argv[1:]- if '--websearch' in command_line_args:- index = command_line_args.index('--websearch')- websearch_value = command_line_args[index + 1]- if(websearch_value == 'bing' or websearch_value == 'google' or websearch_value == 'ddgs'):- config.set_web_search(websearch_value)- else:- return logger.error(f"Invalid websearch value parameter: {websearch_value}")- else:- logger.info("No --websearch argument provided. Using default duckduckgo search.")def init_devika():config = Config()@@ -36,9 +21,6 @@ def init_devika():os.makedirs(projects_dir, exist_ok=True)os.makedirs(logs_dir, exist_ok=True)- init_cmd()- logger.info(f"Using {config.get_web_search()} as default if not specified in the request.")-from src.bert.sentence import SentenceBertlogger.info("Loading sentence-transformer BERT models...")diff --git a/src/llm/llm.py b/src/llm/llm.pyindex ef8534e..023e70b 100644--- a/src/llm/llm.py+++ b/src/llm/llm.py@@ -18,6 +18,7 @@ TIKTOKEN_ENC = tiktoken.get_encoding("cl100k_base")ollama = Ollama()logger = Logger()+agentState = AgentState()class LLM:@@ -46,14 +47,14 @@ class LLM:],"GROQ": [("GROQ Mixtral", "mixtral-8x7b-32768"),- ("GROQ LLAMA2-70B", "llama2-70b-4096"),- ("GROQ GEMMA-7B-IT", "gemma-7b-it"),+ ("GROQ LLAMA2 70B", "llama2-70b-4096"),+ ("GROQ GEMMA 7B IT", "gemma-7b-it"),],- "OLLAMA_MODELS": []+ "OLLAMA": []}- if ollama:- self.models["OLLAMA_MODELS"] = [(model["name"].split(":")[0], model["name"]) for model in- ollama.list_models()]+ if ollama.client:+ self.models["OLLAMA"] = [(model["name"].split(":")[0], model["name"]) for model in+ ollama.models]def list_models(self) -> dict:return self.models@@ -68,8 +69,10 @@ class LLM:@staticmethoddef update_global_token_usage(string: str, project_name: str):token_usage = len(TIKTOKEN_ENC.encode(string))- emit_agent("tokens", {"token_usage": token_usage})- AgentState().update_token_usage(project_name, token_usage)+ agentState.update_token_usage(project_name, token_usage)++ total = agentState.get_latest_token_usage(project_name) + token_usage+ emit_agent("tokens", {"token_usage": total})def inference(self, prompt: str, project_name: str) -> str:self.update_global_token_usage(prompt, project_name)@@ -79,7 +82,7 @@ class LLM:raise ValueError(f"Model {self.model_id} not supported")model_mapping = {- "OLLAMA_MODELS": ollama,+ "OLLAMA": ollama,"CLAUDE": Claude(),"OPENAI": OpenAi(),"GOOGLE": Gemini(),diff --git a/src/llm/ollama_client.py b/src/llm/ollama_client.pyindex 963eb4c..6498d7a 100644--- a/src/llm/ollama_client.py+++ b/src/llm/ollama_client.py@@ -8,17 +8,14 @@ log = Logger()class Ollama:def __init__(self):try:- self.client = ollama.Client(host=Config().get_ollama_api_endpoint())+ self.client = ollama.Client(Config().get_ollama_api_endpoint())+ self.models = self.client.list()["models"]log.info("Ollama available")except:self.client = Nonelog.warning("Ollama not available")log.warning("run ollama server to use ollama models otherwise use other models")- def list_models(self) -> list[dict]:- models = self.client.list()["models"]- return models-def inference(self, model_id: str, prompt: str) -> str:response = self.client.generate(model=model_id,diff --git a/src/logger.py b/src/logger.pyindex 6ff7660..1a7a693 100644--- a/src/logger.py+++ b/src/logger.py@@ -51,8 +51,6 @@ def route_logger(logger: Logger):@wraps(func)def wrapper(*args, **kwargs):- if logger.mode == "off":- return func(*args, **kwargs)# Log entry pointif log_enabled:logger.info(f"{request.path} {request.method}")diff --git a/src/state.py b/src/state.pyindex 6ead64f..80b03dd 100644--- a/src/state.py+++ b/src/state.py@@ -132,7 +132,6 @@ class AgentState:session.commit()emit_agent("agent-state", state_stack)-def is_agent_completed(self, project: str):with Session(self.engine) as session:agent_state = session.query(AgentStateModel).filter(AgentStateModel.project == project).first()@@ -143,7 +142,6 @@ class AgentState:def update_token_usage(self, project: str, token_usage: int):with Session(self.engine) as session:agent_state = session.query(AgentStateModel).filter(AgentStateModel.project == project).first()- print(agent_state)if agent_state:state_stack = json.loads(agent_state.state_stack_json)state_stack[-1]["token_usage"] += token_usagediff --git a/test.py b/test.pynew file mode 100644index 0000000..e7cb4d6--- /dev/null+++ b/test.py@@ -0,0 +1,174 @@+from itertools import islice++import re+from urllib.parse import unquote+from html import unescape+from curl_cffi import requests as curl_requests+import orjson++# class DuckDuckGoSearch1:+# def __init__(self):+# self.query_result = None+# from curl_cffi import requests as curl_requests+# self.asession = curl_requests.Session(+# impersonate="chrome",+# allow_redirects=False,+# )+# self.asession.headers["Referer"] = "https://duckduckgo.com/"+#+# def extract_vqd(self, html_bytes: bytes) -> str:+# """Extract vqd from html bytes."""+# for c1, c1_len, c2 in (+# (b'vqd="', 5, b'"'),+# (b"vqd=", 4, b"&"),+# (b"vqd='", 5, b"'"),+# ):+# try:+# start = html_bytes.index(c1) + c1_len+# end = html_bytes.index(c2, start)+# return html_bytes[start:end].decode()+# except ValueError:+# pass+#+# def text_extract_json(self, html_bytes, keywords):+# """text(backend="api") -> extract json from html."""+# try:+# import orjson+# start = html_bytes.index(b"DDG.pageLayout.load('d',") + 24+# end = html_bytes.index(b");DDG.duckbar.load(", start)+# data = html_bytes[start:end]+# result = orjson.loads(data)+# return result+# except Exception as ex:+# print(f"_text_extract_json() {keywords=} {type(ex).__name__}: {ex}")+#+# def duck(self, query):+# resp = self.asession.request("POST", "https://duckduckgo.com/", data={"q": query})+# resp_content = resp.content+# vqd = self.extract_vqd(resp_content)+#+# payload = {+# "q": query,+# "kl": 'en-us',+# "p": "1",+# "s": "0",+# "df": "",+# "vqd": vqd,+# "ex": "",+# }+#+# resp1 = self.asession.request("GET", "https://links.duckduckgo.com/d.js", params=payload)+# resp_content1 = resp1.content+# page_data = self.text_extract_json(resp_content1, query)+# priority = 0+#+# results = [None] * 1100+#+# for row in page_data:+# href = row.get("u", None)+# if href and href != f"http://www.google.com/search?q={query}":+# body = self.normalize(row["a"])+# if body:+# priority += 1+# result = {+# "title": self.normalize(row["t"]),+# "href": self.normalize_url(href),+# "body": body,+# }+# results[priority] = result+#+# print(list(islice(filter(None, results), 1)))+#+# def normalize_url(self, url: str) -> str:+# from urllib.parse import unquote+#+# """Unquote URL and replace spaces with '+'."""+# return unquote(url.replace(" ", "+")) if url else ""+#+# def normalize(self, raw_html: str) -> str:+# """Strip HTML tags from the raw_html string."""+# from html import unescape+# import re+# REGEX_STRIP_TAGS = re.compile("<.*?>")+#+# return unescape(REGEX_STRIP_TAGS.sub("", raw_html)) if raw_html else ""+#+# def search(self, query):+# self.duck(query)+#+# class DuckDuckGoSearch:+# """DuckDuckGo search engine class.+# methods are inherited from the duckduckgo_search package.+# do not change the methods.+#+# currently, the package is not working with our current setup.+# """+#+# def __init__(self):+# self.query_result = None+# self.asession = curl_requests.Session(impersonate="chrome", allow_redirects=False)+# self.asession.headers["Referer"] = "https://duckduckgo.com/"+#+# def duck(self, query):+# resp = self.asession.request("POST", "https://duckduckgo.com/", data={"q": query})+# vqd = self.extract_vqd(resp.content)+#+# params = {"q": query, "kl": 'en-us', "p": "1", "s": "0", "df": "", "vqd": vqd, "ex": ""}+# resp = self.asession.request("GET", "https://links.duckduckgo.com/d.js", params=params)+# page_data = self.text_extract_json(resp.content)+#+# results = []+# for row in page_data:+# href = row.get("u")+# if href and href != f"http://www.google.com/search?q={query}":+# result = {+# "title": self.normalize(row["t"]),+# "href": self.normalize_url(href),+# "body": self.normalize(row["a"]),+# }+# results.append(result)+#+# self.query_result = results+#+# def search(self, query):+# self.duck(query)+#+# def get_first_link(self):+# print(self.query_result)+# return self.query_result[0]["href"]+#+# @staticmethod+# def extract_vqd(html_bytes: bytes) -> str:+# patterns = [(b'vqd="', 5, b'"'), (b"vqd=", 4, b"&"), (b"vqd='", 5, b"'")]+# for start_pattern, offset, end_pattern in patterns:+# try:+# start = html_bytes.index(start_pattern) + offset+# end = html_bytes.index(end_pattern, start)+# return html_bytes[start:end].decode()+# except ValueError:+# continue+#+# @staticmethod+# def text_extract_json(html_bytes):+# try:+# start = html_bytes.index(b"DDG.pageLayout.load('d',") + 24+# end = html_bytes.index(b");DDG.duckbar.load(", start)+# return orjson.loads(html_bytes[start:end])+# except Exception as ex:+# print(f"Error extracting JSON: {type(ex).__name__}: {ex}")+#+# @staticmethod+# def normalize_url(url: str) -> str:+# return unquote(url.replace(" ", "+")) if url else ""+#+# @staticmethod+# def normalize(raw_html: str) -> str:+# return unescape(re.sub("<.*?>", "", raw_html)) if raw_html else ""+++if __name__ == "__main__":+ # ddgs = DuckDuckGoSearch1()+ # ddgs.search("python")+ from duckduckgo_search import DDGS+ a = DDGS().text("python")+ print(a)\ No newline at end of filediff --git a/ui/src/lib/api.js b/ui/src/lib/api.jsindex 1541bba..67bc599 100644--- a/ui/src/lib/api.js+++ b/ui/src/lib/api.js@@ -1,11 +1,13 @@import {agentState,internet,+ modelList,+ projectList,+ messages,searchEngineList,} from "./store";import { io } from "socket.io-client";-export const socket = io(API_BASE_URL);const getApiBaseUrl = () => {if (typeof window !== 'undefined') {@@ -23,6 +25,7 @@ const getApiBaseUrl = () => {};export const API_BASE_URL = import.meta.env.VITE_API_BASE_URL || getApiBaseUrl();+export const socket = io(API_BASE_URL);export async function fetchInitialData() {diff --git a/ui/src/lib/components/BrowserWidget.svelte b/ui/src/lib/components/BrowserWidget.svelteindex 9b46962..00bf982 100644--- a/ui/src/lib/components/BrowserWidget.svelte+++ b/ui/src/lib/components/BrowserWidget.svelte@@ -16,7 +16,7 @@class="flex-grow border-2 rounded-lg p-2 overflow-x-auto"placeholder="chrome://newtab"value={$agentState?.browser_session.url || ""}- readonly+/></div><div id="browser-content" class="flex-grow overflow-y-auto">diff --git a/ui/src/lib/components/ControlPanel.svelte b/ui/src/lib/components/ControlPanel.svelteindex 88be66a..d17cd2d 100644--- a/ui/src/lib/components/ControlPanel.svelte+++ b/ui/src/lib/components/ControlPanel.svelte@@ -1,7 +1,7 @@<script>import { onMount } from "svelte";- import { projectList, modelList, internet, tokenUsage, agentState, searchEngineList} from "$lib/store";- import { createProject, fetchMessages, fetchInitialData, deleteProject} from "$lib/api";+ import { projectList, modelList, internet, tokenUsage, agentState, messages, searchEngineList} from "$lib/store";+ import { createProject, fetchMessages, fetchInitialData, deleteProject, fetchAgentState} from "$lib/api";import { get } from "svelte/store";let selectedProject;@@ -10,12 +10,14 @@const checkListAndSetItem = (list, itemKey, defaultItem) => {if (get(list) && get(list).length > 0) {- return localStorage.getItem(itemKey);+ const item = localStorage.getItem(itemKey);+ return item ? item : defaultItem;} else {localStorage.setItem(itemKey, "");return defaultItem;}};+selectedProject = checkListAndSetItem( projectList, "selectedProject", "Select Project");selectedModel = checkListAndSetItem( modelList, "selectedModel", "Select Model");selectedSearchEngine = checkListAndSetItem( searchEngineList, "selectedSearchEngine", "Select Search Engine");@@ -25,6 +27,7 @@selectedProject = project;localStorage.setItem("selectedProject", project);fetchMessages();+ fetchAgentState();document.getElementById("project-dropdown").classList.add("hidden");}function selectModel(model) {@@ -49,8 +52,11 @@if (confirm(`Are you sure you want to delete ${project}?`)) {await deleteProject(project);await fetchInitialData();- agentState.setItem(null);- selectProject("Select Project");+ messages.set([]);+ agentState.set(null);+ tokenUsage.set(0);+ selectedProject = "Select Project";+ localStorage.setItem("selectedProject", "");}}@@ -65,6 +71,8 @@const buttonElement = document.getElementById(button);if (+ dropdownElement &&+ buttonElement &&!dropdownElement.contains(event.target) &&!buttonElement.contains(event.target)) {@@ -72,8 +80,7 @@}});}-- onMount(async () => {+ onMount(() => {dropdowns.forEach(({ dropdown, button }) => {document.getElementById(button).addEventListener("click", function () {const dropdownElement = document.getElementById(dropdown);@@ -119,12 +126,10 @@{#if $projectList !== null}{#each $projectList as project}<div- class="flex items-center px-4 hover:bg-gray-300 transition-colors- {selectedProject === project ? 'bg-gray-300' : ''}"- >+ class="flex items-center px-4 hover:bg-gray-200 transition-colors"><buttonhref="#"- class="flex gap-2 items-center text-sm py-3 w-full text-clip"+ class="flex gap-2 items-center text-sm py-3 w-full h-full overflow-x-visible"on:click|preventDefault={() => selectProject(project)}>{project}@@ -177,7 +182,7 @@class="absolute left-0 z-10 mt-2 w-40 origin-top-right rounded-md bg-gray-100 shadow-lg max-h-96 overflow-y-auto hidden"role="menu"aria-orientation="vertical"- aria-labelledby="model-button"+ aria-labelledby="search-engine-button"tabindex="-1"><div role="none" class="flex flex-col divide-y-2 w-full">diff --git a/ui/src/lib/components/MessageContainer.svelte b/ui/src/lib/components/MessageContainer.svelteindex 6433558..10cf57e 100644--- a/ui/src/lib/components/MessageContainer.svelte+++ b/ui/src/lib/components/MessageContainer.svelte@@ -11,7 +11,7 @@previousMessageCount = $messages.length;}});- console.log($messages);+</script><divdiff --git a/ui/src/lib/components/Sidebar.svelte b/ui/src/lib/components/Sidebar.svelteindex 2d69ac8..88f5a7b 100644--- a/ui/src/lib/components/Sidebar.svelte+++ b/ui/src/lib/components/Sidebar.svelte@@ -6,7 +6,7 @@];</script>-<div class="flex flex-col p-4 gap-4 items-center bg-gray-100">+<div class="flex flex-col mx-2 my-4 gap-4 items-center bg-gray-200 rounded-lg py-4 px-2">{#each navItems as item (item.tooltip)}<a href = {item.route}><div class="p-4 rounded nav-button relative">diff --git a/ui/src/lib/components/TerminalWidget.svelte b/ui/src/lib/components/TerminalWidget.svelteindex 68e4530..5524455 100644--- a/ui/src/lib/components/TerminalWidget.svelte+++ b/ui/src/lib/components/TerminalWidget.svelte@@ -6,6 +6,9 @@let terminalElement;let terminal;let fitAddon;+ // agentState.subscribe((value) => {+ // $agentState = value;+ // });onMount(async () => {let xterm = await import('xterm');@@ -51,6 +54,10 @@previousState = { command, output, title };}}+ else {+ // Reset the terminal+ terminal.reset();+ }fitAddon.fit();});@@ -91,21 +98,5 @@overflow-y: auto !important;height: 100%;}-- #terminal-content::-webkit-scrollbar,- #terminal-content :global(.xterm-viewport)::-webkit-scrollbar {- width: 4px;- }-- #terminal-content::-webkit-scrollbar-track,- #terminal-content :global(.xterm-viewport)::-webkit-scrollbar-track {- background: #020617;- border-radius: 10px;- }-- #terminal-content::-webkit-scrollbar-thumb,- #terminal-content :global(.xterm-viewport)::-webkit-scrollbar-thumb {- background: #000000;- border-radius: 10px;- }+</style>diff --git a/ui/src/lib/components/ui/Dropdown.svelte b/ui/src/lib/components/ui/Dropdown.sveltedeleted file mode 100644index 43daa1f..0000000--- a/ui/src/lib/components/ui/Dropdown.svelte+++ /dev/null@@ -1,115 +0,0 @@-<script>- /**- * @type {string} The text shown when no option is selected- */- export let label;-- /**- * @type {string | null | undefined} The ID of the selected entry or null- */- export let selection = undefined;-- /**- * @type {Record<string, string>} An object of options with the key as the ID and the value as the label- */- export let options;-- /** @type {HTMLElement | null} **/- let dropdown = null;-- let open = false;-- /**- * @param {string} id ID of the selected element- */- function selectElement(id) {- selection = id;- open = false;- }-- /**- * Svelte Action function to handle clicks outside of the given element- * @param {HTMLElement} element The dropdown element- * @param {() => void} callbackFunction The function to call when a click outside is detected- */- function onClickOutside(element, callbackFunction) {- function onClick(event) {- if (!element.contains(event.target)) {- callbackFunction();- }- }-- let registered = false;-- setTimeout(() => {- document.body.addEventListener('click', onClick);- registered = true;- }, 100)-- return {- update(newCallbackFunction) {- callbackFunction = newCallbackFunction;- },- destroy() {- if (registered) {- document.body.removeEventListener('click', onClick);- registered = false;- }- }- }- }-- // This needs to be a function so it can passed as slot prop- function closeDropdown() {- open = false;- }-</script>--<div class="dropdown-menu relative inline-block">- <button- type="button"- class="inline-flex justify-center w-full gap-x-1.5 rounded-md bg-slate-900 px-3 py-2 text-sm font-semibold text-white shadow-sm ring-1 ring-inset ring-indigo-700 hover:bg-slate-800"- aria-expanded="true"- aria-haspopup="true"- on:click={() => open = !open}- >- <span>- {selection ? options[selection] : label}- </span>-- <svg- class="-mr-1 h-5 w-5 text-gray-400"- viewBox="0 0 20 20"- fill="currentColor"- aria-hidden="true"- >- <path- fill-rule="evenodd"- d="M5.23 7.21a.75.75 0 011.06.02L10 11.168l3.71-3.938a.75.75 0 111.08 1.04l-4.25 4.5a.75.75 0 01-1.08 0l-4.25-4.5a.75.75 0 01.02-1.06z"- clip-rule="evenodd"- />- </svg>- </button>- {#if open}- <div- bind:this={dropdown}- class="absolute left-0 z-10 w-fit origin-top-left rounded-md bg-slate-800 shadow-lg ring-1 ring-indigo-700 ring-opacity-5 focus:outline-none"- role="menu"- tabindex="-1"- use:onClickOutside={() => { open = false; }}- >- <div role="none">- <slot name="prefix-entries" {closeDropdown} />-- {#each Object.entries(options) as [id, label]}- <button- class="text-white block px-4 py-2 text-sm hover:bg-slate-700 w-full text-left overflow-clip"- on:click|preventDefault={() => selectElement(id)}- >- {label}- </button>- {/each}- </div>- </div>- {/if}-</div>diff --git a/ui/src/routes/settings/+page.svelte b/ui/src/routes/settings/+page.svelteindex f56e2f3..5490393 100644--- a/ui/src/routes/settings/+page.svelte+++ b/ui/src/routes/settings/+page.svelte@@ -4,13 +4,24 @@let settings = {};let editMode = false;+ let original = {};onMount(async () => {settings = await fetchSettings();+ // make a copy of the original settings+ original = JSON.parse(JSON.stringify(settings));+});const save = async () => {- await updateSettings({"API_KEYS": settings["API_KEYS"]});+ let updated = {};+ for (let key in settings) {+ if (settings[key] !== original[key]) {+ updated[key] = settings[key];+ }+ }+ await updateSettings(updated);+editMode = !editMode;};@@ -19,48 +30,97 @@};</script>-<div class="p-4 h-full w-full gap-8 flex flex-col">+<div class="p-4 h-full w-full gap-8 flex flex-col overflow-y-auto"><h1 class="text-3xl">Settings</h1><div class="flex flex-col w-full">{#if settings["API_KEYS"]}- <div class="flex flex-col gap-4">- <h2 class="text-xl">API Keys</h2>- <div class="flex flex-col gap-4">- {#each Object.entries(settings["API_KEYS"]) as [key, value]}- <div class="flex gap-3 items-center">- <p class="w-20">{key}</p>- <input- type="text"- bind:value={settings["API_KEYS"][key]}- name={key}- class="w-1/3 p-2 border-2 rounded-lg {editMode ? '' : 'bg-gray-100 text-gray-500'}"- readonly={!editMode}- />- </div>- {/each}+ <div class="flex gap-4 w-full">++ <div class="flex flex-col gap-4 w-full">+ <h2 class="text-xl">API Keys</h2>+ <div class="flex flex-col gap-4">+ {#each Object.entries(settings["API_KEYS"]) as [key, value]}+ <div class="flex gap-1 items-center">+ <p class="w-48 text-sm">{key.toLowerCase()}</p>+ <input+ type="text"+ bind:value={settings["API_KEYS"][key]}+ name={key}+ class="p-2 border-2 w-1/2 rounded-lg {editMode ? '' : 'bg-gray-100 text-gray-500'}"+ readonly={!editMode}+ on:change={settings["API_KEYS"][key] = $event.target.value}+ />+ </div>+ {/each}+ </div></div>- <!-- edit and save button -->- <div class="flex gap-4">- {#if !editMode}- <button- id="btn-edit"- class="p-2 border-2 rounded-lg flex gap-3 items-center hover:bg-gray-200"- on:click={edit}- >- <i class="fas fa-edit"></i>- Edit- </button>- {:else}- <button- id="btn-save"- class="p-2 border-2 rounded-lg flex gap-3 items-center hover:bg-gray-200"- on:click={save}- >- <i class="fas fa-save"></i>- Save- </button>- {/if}++ <div class="flex flex-col gap-8 w-full">++ <div class="flex flex-col gap-2">+ <h2 class="text-xl">API Endpoints</h2>+ <div class="flex flex-col gap-4">+ {#each Object.entries(settings["API_ENDPOINTS"]) as [key, value]}+ <div class="flex gap-3 items-center">+ <p class="w-28">{key.toLowerCase()}</p>+ <input+ type="text"+ bind:value={settings["API_ENDPOINTS"][key]}+ name={key}+ class="p-2 w-1/2 border-2 rounded-lg {editMode ? '' : 'bg-gray-100 text-gray-500'}"+ readonly={!editMode}+ on:change={settings["API_ENDPOINTS"][key] = $event.target.value}+ />+ </div>+ {/each}+ </div>++ </div>+ <div class="flex flex-col gap-2">+ <h2 class="text-xl">Logging</h2>+ <div class="flex flex-col gap-4">+ {#each Object.entries(settings["LOGGING"]) as [key, value]}+ <div class="flex gap-3 items-center">+ <p class="w-28">{key.toLowerCase()}</p>+ <input+ type="text"+ bind:value={settings["LOGGING"][key]}+ name={key}+ class="p-2 border-2 rounded-lg {editMode ? '' : 'bg-gray-100 text-gray-500'}"+ readonly={!editMode}+ placeholder="true/false"+ on:change={settings["LOGGING"][key] = $event.target.value}+ />+ </div>+ {/each}+ </div>++ </div>++ <!-- edit and save button -->+ <div class="flex gap-4">+ {#if !editMode}+ <button+ id="btn-edit"+ class="p-2 border-2 rounded-lg flex gap-3 items-center hover:bg-gray-200"+ on:click={edit}+ >+ <i class="fas fa-edit"></i>+ Edit+ </button>+ {:else}+ <button+ id="btn-save"+ class="p-2 border-2 rounded-lg flex gap-3 items-center hover:bg-gray-200"+ on:click={save}+ >+ <i class="fas fa-save"></i>+ Save+ </button>+ {/if}+ </div></div>+</div>{/if}</div>diff --git a/ui/src/routes/settings/Settings/Button.svelte b/ui/src/routes/settings/Settings/Button.sveltedeleted file mode 100644index b0ba418..0000000--- a/ui/src/routes/settings/Settings/Button.svelte+++ /dev/null@@ -1,21 +0,0 @@-<script lang="ts">- import clsx from "clsx";- export let isActive: boolean = undefined;- export let disabled: boolean = undefined;- export let onClick: () => void = undefined;- export let activeText: string = undefined;- export let text: string;- export let type: "button" | "submit" | "reset" = undefined;-</script>--<button- class={clsx(- `px-4 py-2 rounded w-full mt-2 ${isActive ? "bg-slate-800" : "bg-indigo-700"}`,- disabled ? "cursor-not-allowed bg-gray-600" : "cursor-pointer"- )}- on:click={onClick}- disabled={isActive || disabled}- {type}->- {@html isActive ? activeText : text}-</button>diff --git a/ui/src/routes/settings/Settings/Input.svelte b/ui/src/routes/settings/Settings/Input.sveltedeleted file mode 100644index c6a8433..0000000--- a/ui/src/routes/settings/Settings/Input.svelte+++ /dev/null@@ -1,57 +0,0 @@-<script lang="ts">- export let type = "text";- export let value = type === "text" ? "" : null;- export let error = "";- export let label = "";- export let placeholder = "";-- function handleInput({ target: t }) {- if (type === "number") {- value = t.value === "" ? null : t.valueAsNumber;- } else {- value = t.value;- }- }-</script>--<label class="block">- {#if label}- <span class="block mb-1">{label}</span>- {/if}- <input- class="block input"- class:error- {type}- {placeholder}- {value}- on:input={handleInput}- on:input- on:blur- />- {#if error}- <span class="block error-text">{error}</span>- {/if}-</label>--<style>- .input {- border-radius: 4px;- padding: 6px 10px;- margin: 0;- color: black;- width: 100%;- }-- .block {- display: block;- }-- .error {- border-color: #f55;- }-- .error-text {- margin-top: 4px;- color: #f55;- }-</style>diff --git a/ui/src/routes/settings/Settings/Settings.svelte b/ui/src/routes/settings/Settings/Settings.sveltedeleted file mode 100644index fe4777f..0000000--- a/ui/src/routes/settings/Settings/Settings.svelte+++ /dev/null@@ -1,78 +0,0 @@-<script lang="ts">- import { onMount } from "svelte";- import { getSettings, setSettings } from "../../../lib/api";- import Button from "./Button.svelte";- import Input from "./Input.svelte";- let result: any;- let settingsLoading: boolean;- onMount(async () => {- result = await getSettings();- // console.log(result);- });-- async function handleSubmit() {- settingsLoading = true;- const postResult = await setSettings(result);- // console.log(postResult);- settingsLoading = false;- formChanged = false;- }- function checkNonEmptyRecursive(obj) {- for (const key in obj) {- if (obj.hasOwnProperty(key)) {- if (typeof obj[key] === "object") {- if (!checkNonEmptyRecursive(obj[key])) {- return false;- }- } else if (!obj[key]) {- return false;- }- }- }- return true;- }- $: formValid = checkNonEmptyRecursive(result);- let formChanged = false;-</script>--<div class="px-4 py-8 max-w-[600px]">- {#if result}- <form on:submit|preventDefault={handleSubmit}>- <div class="space-y-8">- {#each Object.keys(result) as mainKey}- <div>- <p class="mb-4">{mainKey}</p>- <div class="pl-4 space-y-3">- {#each Object.keys(result[mainKey]) as key}- <Input- label={key}- type="text"- bind:value={result[mainKey][key]}- on:input={() => {- formChanged = true;- }}- error={!result[mainKey][key] ? "field can't be empty" : ""}- />- {/each}- </div>- </div>- {/each}- </div>- <div class="h-[20px]"></div>- {#if !formValid}- <p class="text-[#f55]">All fields must be filled</p>- <div class="h-[10px]"></div>- {/if}-- {#if formChanged}- <Button- type="submit"- text="Save"- isActive={settingsLoading}- activeText="Saving..."- disabled={!formValid}- ></Button>- {/if}- </form>- {/if}-</div>