Agent Zero AI framework
Instruments
frdela6d0c5b86218
diff --git a/.gitignore b/.gitignoreindex 4e03ec2c6..1deed9b2a 100644--- a/.gitignore+++ b/.gitignore@@ -38,4 +38,18 @@ knowledge/**/*.*# Explicitly allow the default folder and its contents!knowledge/default/-!knowledge/default/**\ No newline at end of file+!knowledge/default/**++# Ignore everything in the "instruments" directory+instruments/*++# Do not ignore subdirectories (so we can track .gitkeep)+!instruments/*/++# Ignore all files within subdirectories (except .gitkeep)+instruments/**/*.*+!instruments/**/.gitkeep++# Explicitly allow the default folder and its contents+!instruments/default/+!instruments/default/**\ No newline at end of filediff --git a/agent.py b/agent.pyindex 2245cf69d..86c0b0ff4 100644--- a/agent.py+++ b/agent.py@@ -111,7 +111,8 @@ class AgentConfig:)code_exec_docker_volumes: dict[str, dict[str, str]] = field(default_factory=lambda: {- files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"}+ files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"},+ files.get_abs_path("instruments"): {"bind": "/instruments", "mode": "rw"},})code_exec_ssh_enabled: bool = Truediff --git a/initialize.py b/initialize.pyindex 18176215d..3c7316d12 100644--- a/initialize.py+++ b/initialize.py@@ -1,5 +1,6 @@import modelsfrom agent import AgentConfig+from python.helpers import filesdef initialize():@@ -47,7 +48,10 @@ def initialize():# code_exec_docker_name = "agent-zero-exe",# code_exec_docker_image = "frdel/agent-zero-exe:latest",# code_exec_docker_ports = { "22/tcp": 50022 }- # code_exec_docker_volumes = { files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"} }+ # code_exec_docker_volumes = {+ # files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"},+ # files.get_abs_path("instruments"): {"bind": "/instruments", "mode": "rw"},+ # },code_exec_ssh_enabled = True,# code_exec_ssh_addr = "localhost",# code_exec_ssh_port = 50022,diff --git a/instruments/custom/.gitkeep b/instruments/custom/.gitkeepnew file mode 100644index 000000000..e69de29bbdiff --git a/instruments/default/.gitkeep b/instruments/default/.gitkeepnew file mode 100644index 000000000..e69de29bbdiff --git a/instruments/default/yt_download/yt_download.md b/instruments/default/yt_download/yt_download.mdnew file mode 100644index 000000000..b469527dd--- /dev/null+++ b/instruments/default/yt_download/yt_download.md@@ -0,0 +1,6 @@+# Problem+Download a YouTube video+# Solution+1. cd to the desired location to download+2. Run instrument "bash /instruments/default/yt_download/yt_download.sh <url>" with your video URL+3. Wait for the terminal to finish\ No newline at end of filediff --git a/instruments/default/yt_download/yt_download.sh b/instruments/default/yt_download/yt_download.shnew file mode 100755index 000000000..a83b6004c--- /dev/null+++ b/instruments/default/yt_download/yt_download.sh@@ -0,0 +1,7 @@+#!/bin/bash++# Install yt-dlp and ffmpeg+sudo apt-get update && sudo apt-get install -y yt-dlp ffmpeg++# Download the best video and audio, and merge them+yt-dlp -f bestvideo+bestaudio --merge-output-format mp4 "$1"\ No newline at end of filediff --git a/knowledge/default/solutions/get_current_time.md b/knowledge/default/solutions/get_current_time.mdnew file mode 100644index 000000000..959dd9dce--- /dev/null+++ b/knowledge/default/solutions/get_current_time.md@@ -0,0 +1,13 @@+# Problem+User asked for current time in timezone+# Solution+Use code_execution_tool with following python code adjusted for your timezone+~~~python+from datetime import datetime+import pytz++timezone = pytz.timezone('America/New_York')+current_time = datetime.now(timezone)++print("Current time in New York:", current_time)+~~~diff --git a/knowledge/default/solutions/yt_download.md b/knowledge/default/solutions/yt_download.mddeleted file mode 100644index 2aaa6ccef..000000000--- a/knowledge/default/solutions/yt_download.md+++ /dev/null@@ -1,6 +0,0 @@-# Problem-Download a YouTube video-# Solution-1. If you don't have exact URL, use knowledge_tool to get it-2. Pip install yt-dlp and ffmpeg-3. Download the video using yt-dlp command: 'yt-dlp YT_URL', replace YT_URL with your video URL.\ No newline at end of filediff --git a/prompts/default/agent.system.instruments.md b/prompts/default/agent.system.instruments.mdnew file mode 100644index 000000000..b9b385a05--- /dev/null+++ b/prompts/default/agent.system.instruments.md@@ -0,0 +1,4 @@+# Instruments+- following are instruments that could possibly be used:++{{instruments}}\ No newline at end of filediff --git a/prompts/default/agent.system.main.solving.md b/prompts/default/agent.system.main.solving.mdindex 62d580f23..1abd5a286 100644--- a/prompts/default/agent.system.main.solving.md+++ b/prompts/default/agent.system.main.solving.md@@ -3,7 +3,7 @@- Explain each step using your thoughts argument.0. Outline the plan by repeating these instructions.-1. Check the memory output of your knowledge_tool. Maybe you have solved similar task before and already have helpful information.+1. Check your memories, solutions and instruments. Prefer using instruments when possible.2. Check the online sources output of your knowledge_tool.- Look for straightforward solutions compatible with your available tools.- Always look for opensource python/nodejs/terminal tools and packages first.diff --git a/prompts/default/agent.system.main.tips.md b/prompts/default/agent.system.main.tips.mdindex 4b3dc567b..9298c6820 100644--- a/prompts/default/agent.system.main.tips.md+++ b/prompts/default/agent.system.main.tips.md@@ -14,6 +14,10 @@- Communication is the key to succesfull solution.- NEVER delegate your whole task, only parts of it.+## Instruments+- Instruments are programs you can utilize to solve tasks+- Instrument descriptions are injected into the prompt and can be executed with code_execution_tool+## Tips and tricks- Focus on python/nodejs/linux libraries when searching for solutions. You can use them with your tools and make solutions easy.- Sometimes you don't need tools, some things can be determined.diff --git a/python/extensions/message_loop_prompts/_50_recall_memories.py b/python/extensions/message_loop_prompts/_50_recall_memories.pyindex a3a1cb951..24fe706bb 100644--- a/python/extensions/message_loop_prompts/_50_recall_memories.py+++ b/python/extensions/message_loop_prompts/_50_recall_memories.py@@ -60,12 +60,12 @@ class RecallMemories(Extension):# log the short resultif not isinstance(memories, list) or len(memories) == 0:log_item.update(- heading="No useful memories found.",+ heading="No useful memories found",)returnelse:log_item.update(- heading=f"\n\n{len(memories)} memories found.",+ heading=f"{len(memories)} memories found",)# concatenate memory.page_content in memories:diff --git a/python/extensions/message_loop_prompts/_51_recall_solutions.py b/python/extensions/message_loop_prompts/_51_recall_solutions.pyindex d05c2500f..4b177a073 100644--- a/python/extensions/message_loop_prompts/_51_recall_solutions.py+++ b/python/extensions/message_loop_prompts/_51_recall_solutions.py@@ -7,7 +7,8 @@ class RecallSolutions(Extension):INTERVAL = 3HISTORY = 5- RESULTS = 3+ SOLUTIONS_COUNT = 2+ INSTRUMENTS_COUNT = 2THRESHOLD = 0.1async def execute(self, loop_data: LoopData = LoopData(), **kwargs):@@ -19,74 +20,78 @@ class RecallSolutions(Extension):async def search_solutions(self, loop_data: LoopData, **kwargs):# try:- # show temp info message- self.agent.context.log.log(- type="info", content="Searching memory for solutions...", temp=True+ # show temp info message+ self.agent.context.log.log(+ type="info", content="Searching memory for solutions...", temp=True+ )++ # show full util message, this will hide temp message immediately if turned on+ log_item = self.agent.context.log.log(+ type="util",+ heading="Searching memory for solutions...",+ )++ # get system message and chat history for util llm+ msgs_text = self.agent.concat_messages(+ self.agent.history[-RecallSolutions.HISTORY :]+ ) # only last X messages+ system = self.agent.read_prompt(+ "memory.solutions_query.sys.md", history=msgs_text+ )++ # log query streamed by LLM+ def log_callback(content):+ log_item.stream(query=content)++ # call util llm to summarize conversation+ query = await self.agent.call_utility_llm(+ system=system, msg=loop_data.message, callback=log_callback+ )++ # get solutions database+ db = await Memory.get(self.agent)++ solutions = await db.search_similarity_threshold(+ query=query,+ limit=RecallSolutions.SOLUTIONS_COUNT,+ threshold=RecallSolutions.THRESHOLD,+ filter=f"area == '{Memory.Area.SOLUTIONS.value}'",+ )+ instruments = await db.search_similarity_threshold(+ query=query,+ limit=RecallSolutions.INSTRUMENTS_COUNT,+ threshold=RecallSolutions.THRESHOLD,+ filter=f"area == '{Memory.Area.INSTRUMENTS.value}'",+ )++ log_item.update(+ heading=f"{len(instruments)} instruments, {len(solutions)} solutions found",+ )++ if instruments:+ instruments_text = ""+ for instrument in instruments:+ instruments_text += instrument.page_content + "\n\n"+ instruments_text = instruments_text.strip()+ log_item.update(instruments=instruments_text)+ instruments_prompt = self.agent.read_prompt(+ "agent.system.instruments.md", instruments=instruments_text)+ loop_data.system.append(instruments_prompt)- # show full util message, this will hide temp message immediately if turned on- log_item = self.agent.context.log.log(- type="util",- heading="Searching memory for solutions...",- )-- # get system message and chat history for util llm- msgs_text = self.agent.concat_messages(- self.agent.history[-RecallSolutions.HISTORY :]- ) # only last X messages- system = self.agent.read_prompt(- "memory.solutions_query.sys.md", history=msgs_text- )-- # log query streamed by LLM- def log_callback(content):- log_item.stream(query=content)-- # call util llm to summarize conversation- query = await self.agent.call_utility_llm(- system=system, msg=loop_data.message, callback=log_callback- )-- # get solutions database- db = await Memory.get(self.agent)-- solutions = await db.search_similarity_threshold(- query=query,- limit=RecallSolutions.RESULTS,- threshold=RecallSolutions.THRESHOLD,- filter=f"area == '{Memory.Area.SOLUTIONS.value}'"- )-- # log the short result- if not isinstance(solutions, list) or len(solutions) == 0:- log_item.update(- heading="No successful solution memories found.",- )- return- else:- log_item.update(- heading=f"\n\n{len(solutions)} successful solution memories found.",- )-- # concatenate solution.page_content in solutions:+ if solutions:solutions_text = ""for solution in solutions:solutions_text += solution.page_content + "\n\n"solutions_text = solutions_text.strip()-- # log the full resultslog_item.update(solutions=solutions_text)-- # place to promptsolutions_prompt = self.agent.read_prompt("agent.system.solutions.md", solutions=solutions_text)-- # append to system messageloop_data.system.append(solutions_prompt)- # except Exception as e:- # err = errors.format_error(e)- # self.agent.context.log.log(- # type="error", heading="Recall solutions extension error:", content=err- # )+ # except Exception as e:+ # err = errors.format_error(e)+ # self.agent.context.log.log(+ # type="error", heading="Recall solutions extension error:", content=err+ # )diff --git a/python/helpers/knowledge_import.py b/python/helpers/knowledge_import.pyindex 7c2e65d0d..7e3b7a8c1 100644--- a/python/helpers/knowledge_import.py+++ b/python/helpers/knowledge_import.py@@ -34,11 +34,14 @@ def calculate_checksum(file_path: str) -> str:def load_knowledge(- log_item: LogItem | None, knowledge_dir: str, index: Dict[str, KnowledgeImport]+ log_item: LogItem | None,+ knowledge_dir: str,+ index: Dict[str, KnowledgeImport],+ metadata: dict[str, Any] = {},+ filename_pattern: str = "**/*",) -> Dict[str, KnowledgeImport]:- knowledge_dir = files.get_abs_path("knowledge",knowledge_dir)- from python.helpers.memory import Memory+ # from python.helpers.memory import Memory# Mapping file extensions to corresponding loader classesfile_types_loaders = {@@ -47,62 +50,67 @@ def load_knowledge("csv": CSVLoader,"html": UnstructuredHTMLLoader,"json": JSONLoader,- "md": UnstructuredMarkdownLoader,+ # "md": UnstructuredMarkdownLoader,+ "md": TextLoader,}cnt_files = 0cnt_docs = 0- for area in Memory.Area:- subdir = files.get_abs_path(knowledge_dir, area.value)+ # for area in Memory.Area:+ # subdir = files.get_abs_path(knowledge_dir, area.value)- if not os.path.exists(subdir):- os.makedirs(subdir)- continue+ # if not os.path.exists(knowledge_dir):+ # os.makedirs(knowledge_dir)+ # continue- # Fetch all files in the directory with specified extensions- kn_files = glob.glob(subdir + "/**/*", recursive=True)- if kn_files:- print(f"Found {len(kn_files)} knowledge files in {subdir}, processing...")- if log_item:- log_item.stream(- progress=f"\nFound {len(kn_files)} knowledge files in {subdir}, processing...",- )+ # Fetch all files in the directory with specified extensions+ kn_files = glob.glob(knowledge_dir + "/" + filename_pattern, recursive=True)+ kn_files = [f for f in kn_files if os.path.isfile(f)]- for file_path in kn_files:- ext = file_path.split(".")[-1].lower()- if ext in file_types_loaders:- checksum = calculate_checksum(file_path)- file_key = file_path # os.path.relpath(file_path, knowledge_dir)-- # Load existing data from the index or create a new entry- file_data = index.get(file_key, {})-- if file_data.get("checksum") == checksum:- file_data["state"] = "original"- else:- file_data["state"] = "changed"-- if file_data["state"] == "changed":- file_data["checksum"] = checksum- loader_cls = file_types_loaders[ext]- loader = loader_cls(- file_path,- **(- text_loader_kwargs- if ext in ["txt", "csv", "html", "md"]- else {}- ),- )- file_data["documents"] = loader.load_and_split()- for doc in file_data["documents"]:- doc.metadata["area"] = area.value- cnt_files += 1- cnt_docs += len(file_data["documents"])- # print(f"Imported {len(file_data['documents'])} documents from {file_path}")-- # Update the index- index[file_key] = file_data # type: ignore+ if kn_files:+ print(+ f"Found {len(kn_files)} knowledge files in {knowledge_dir}, processing..."+ )+ if log_item:+ log_item.stream(+ progress=f"\nFound {len(kn_files)} knowledge files in {knowledge_dir}, processing...",+ )++ for file_path in kn_files:+ ext = file_path.split(".")[-1].lower()+ if ext in file_types_loaders:+ checksum = calculate_checksum(file_path)+ file_key = file_path # os.path.relpath(file_path, knowledge_dir)++ # Load existing data from the index or create a new entry+ file_data = index.get(file_key, {})++ if file_data.get("checksum") == checksum:+ file_data["state"] = "original"+ else:+ file_data["state"] = "changed"++ if file_data["state"] == "changed":+ file_data["checksum"] = checksum+ loader_cls = file_types_loaders[ext]+ loader = loader_cls(+ file_path,+ **(+ text_loader_kwargs+ if ext in ["txt", "csv", "html", "md"]+ else {}+ ),+ )+ file_data["documents"] = loader.load_and_split()+ for doc in file_data["documents"]:+ doc.metadata = {**doc.metadata, **metadata}+ cnt_files += 1+ cnt_docs += len(file_data["documents"])+ # print(f"Imported {len(file_data['documents'])} documents from {file_path}")++ # Update the index+ index[file_key] = file_data # type: ignore# loop index where state is not set and mark it as removedfor file_key, file_data in index.items():diff --git a/python/helpers/memory.py b/python/helpers/memory.pyindex 417ef4790..fb14508e0 100644--- a/python/helpers/memory.py+++ b/python/helpers/memory.py@@ -21,20 +21,23 @@ from python.helpers.log import Log, LogItemfrom enum import Enumfrom agent import Agent+class MyFaiss(FAISS):- #override aget_by_ids+ # override aget_by_idsdef get_by_ids(self, ids: Sequence[str], /) -> List[Document]:# return all self.docstore._dict[id] in ids- return [self.docstore._dict[id] for id in ids if id in self.docstore._dict] #type: ignore+ return [self.docstore._dict[id] for id in ids if id in self.docstore._dict] # type: ignoreasync def aget_by_ids(self, ids: Sequence[str], /) -> List[Document]:return self.get_by_ids(ids)+class Memory:class Area(Enum):MAIN = "main"SOLUTIONS = "solutions"+ INSTRUMENTS = "instruments"index: dict[str, "MyFaiss"] = {}@@ -130,7 +133,7 @@ class Memory:# normalize_L2=True,relevance_score_fn=Memory._cosine_normalizer,)- return db # type: ignore+ return db # type: ignoredef __init__(self,@@ -160,8 +163,8 @@ class Memory:with open(index_path, "r") as f:index = json.load(f)- for kn_dir in kn_dirs:- index = knowledge_import.load_knowledge(log_item, kn_dir, index)+ # preload knowledge folders+ index = self._preload_knowledge_folders(log_item, kn_dirs, index)for file in index:if index[file]["state"] in ["changed", "removed"] and index[file].get(@@ -187,6 +190,33 @@ class Memory:with open(index_path, "w") as f:json.dump(index, f)+ def _preload_knowledge_folders(+ self,+ log_item: LogItem | None,+ kn_dirs: list[str],+ index: dict[str, knowledge_import.KnowledgeImport],+ ):+ # load knowledge folders, subfolders by area+ for kn_dir in kn_dirs:+ for area in Memory.Area:+ index = knowledge_import.load_knowledge(+ log_item,+ files.get_abs_path("knowledge", kn_dir, area.value),+ index,+ {"area": area.value},+ )++ # load instruments descriptions+ index = knowledge_import.load_knowledge(+ log_item,+ files.get_abs_path("instruments"),+ index,+ {"area": Memory.Area.INSTRUMENTS.value},+ filename_pattern="**/*.md",+ )++ return index+async def search_similarity_threshold(self, query: str, limit: int, threshold: float, filter: str = ""):@@ -235,9 +265,9 @@ class Memory:async def delete_documents_by_ids(self, ids: list[str]):# aget_by_ids is not yet implemented in faiss, need to do a workaround- rem_docs =self.db.get_by_ids(ids) # existing docs to remove (prevents error)+ rem_docs = self.db.get_by_ids(ids) # existing docs to remove (prevents error)if rem_docs:- rem_ids = [doc.metadata["id"] for doc in rem_docs] # ids to remove+ rem_ids = [doc.metadata["id"] for doc in rem_docs] # ids to removeawait self.db.adelete(ids=rem_ids)if rem_docs:diff --git a/python/tools/code_execution_tool.py b/python/tools/code_execution_tool.pyindex eae0a7ba6..78c071be1 100644--- a/python/tools/code_execution_tool.py+++ b/python/tools/code_execution_tool.py@@ -36,7 +36,7 @@ class CodeExecution(Tool):response = await self.execute_terminal_command(self.args["code"])elif runtime == "output":response = await self.get_terminal_output(- wait_with_output=5, wait_without_output=20+ wait_with_output=5, wait_without_output=60)elif runtime == "reset":response = await self.reset_terminal()@@ -137,7 +137,7 @@ class CodeExecution(Tool):return await self.get_terminal_output()async def get_terminal_output(- self, wait_with_output=3, wait_without_output=10, max_exec_time=15+ self, wait_with_output=3, wait_without_output=10, max_exec_time=60):idle = 0SLEEP_TIME = 0.1