Skip to content

Agent Zero AI framework

Instruments

frdela6d0c5b86218
diff --git a/.gitignore b/.gitignore
index 4e03ec2c6..1deed9b2a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,4 +38,18 @@ knowledge/**/*.*
# Explicitly allow the default folder and its contents
!knowledge/default/
-!knowledge/default/**
\ No newline at end of file
+!knowledge/default/**
+
+# Ignore everything in the "instruments" directory
+instruments/*
+
+# Do not ignore subdirectories (so we can track .gitkeep)
+!instruments/*/
+
+# Ignore all files within subdirectories (except .gitkeep)
+instruments/**/*.*
+!instruments/**/.gitkeep
+
+# Explicitly allow the default folder and its contents
+!instruments/default/
+!instruments/default/**
\ No newline at end of file
diff --git a/agent.py b/agent.py
index 2245cf69d..86c0b0ff4 100644
--- a/agent.py
+++ b/agent.py
@@ -111,7 +111,8 @@ class AgentConfig:
)
code_exec_docker_volumes: dict[str, dict[str, str]] = field(
default_factory=lambda: {
- files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"}
+ files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"},
+ files.get_abs_path("instruments"): {"bind": "/instruments", "mode": "rw"},
}
)
code_exec_ssh_enabled: bool = True
diff --git a/initialize.py b/initialize.py
index 18176215d..3c7316d12 100644
--- a/initialize.py
+++ b/initialize.py
@@ -1,5 +1,6 @@
import models
from agent import AgentConfig
+from python.helpers import files
def initialize():
@@ -47,7 +48,10 @@ def initialize():
# code_exec_docker_name = "agent-zero-exe",
# code_exec_docker_image = "frdel/agent-zero-exe:latest",
# code_exec_docker_ports = { "22/tcp": 50022 }
- # code_exec_docker_volumes = { files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"} }
+ # code_exec_docker_volumes = {
+ # files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"},
+ # files.get_abs_path("instruments"): {"bind": "/instruments", "mode": "rw"},
+ # },
code_exec_ssh_enabled = True,
# code_exec_ssh_addr = "localhost",
# code_exec_ssh_port = 50022,
diff --git a/instruments/custom/.gitkeep b/instruments/custom/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/instruments/default/.gitkeep b/instruments/default/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/instruments/default/yt_download/yt_download.md b/instruments/default/yt_download/yt_download.md
new file mode 100644
index 000000000..b469527dd
--- /dev/null
+++ b/instruments/default/yt_download/yt_download.md
@@ -0,0 +1,6 @@
+# Problem
+Download a YouTube video
+# Solution
+1. cd to the desired location to download
+2. Run instrument "bash /instruments/default/yt_download/yt_download.sh <url>" with your video URL
+3. Wait for the terminal to finish
\ No newline at end of file
diff --git a/instruments/default/yt_download/yt_download.sh b/instruments/default/yt_download/yt_download.sh
new file mode 100755
index 000000000..a83b6004c
--- /dev/null
+++ b/instruments/default/yt_download/yt_download.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Install yt-dlp and ffmpeg
+sudo apt-get update && sudo apt-get install -y yt-dlp ffmpeg
+
+# Download the best video and audio, and merge them
+yt-dlp -f bestvideo+bestaudio --merge-output-format mp4 "$1"
\ No newline at end of file
diff --git a/knowledge/default/solutions/get_current_time.md b/knowledge/default/solutions/get_current_time.md
new file mode 100644
index 000000000..959dd9dce
--- /dev/null
+++ b/knowledge/default/solutions/get_current_time.md
@@ -0,0 +1,13 @@
+# Problem
+User asked for current time in timezone
+# Solution
+Use code_execution_tool with following python code adjusted for your timezone
+~~~python
+from datetime import datetime
+import pytz
+
+timezone = pytz.timezone('America/New_York')
+current_time = datetime.now(timezone)
+
+print("Current time in New York:", current_time)
+~~~
diff --git a/knowledge/default/solutions/yt_download.md b/knowledge/default/solutions/yt_download.md
deleted file mode 100644
index 2aaa6ccef..000000000
--- a/knowledge/default/solutions/yt_download.md
+++ /dev/null
@@ -1,6 +0,0 @@
-# Problem
-Download a YouTube video
-# Solution
-1. If you don't have exact URL, use knowledge_tool to get it
-2. Pip install yt-dlp and ffmpeg
-3. Download the video using yt-dlp command: 'yt-dlp YT_URL', replace YT_URL with your video URL.
\ No newline at end of file
diff --git a/prompts/default/agent.system.instruments.md b/prompts/default/agent.system.instruments.md
new file mode 100644
index 000000000..b9b385a05
--- /dev/null
+++ b/prompts/default/agent.system.instruments.md
@@ -0,0 +1,4 @@
+# Instruments
+- following are instruments that could possibly be used:
+
+{{instruments}}
\ No newline at end of file
diff --git a/prompts/default/agent.system.main.solving.md b/prompts/default/agent.system.main.solving.md
index 62d580f23..1abd5a286 100644
--- a/prompts/default/agent.system.main.solving.md
+++ b/prompts/default/agent.system.main.solving.md
@@ -3,7 +3,7 @@
- Explain each step using your thoughts argument.
0. Outline the plan by repeating these instructions.
-1. Check the memory output of your knowledge_tool. Maybe you have solved similar task before and already have helpful information.
+1. Check your memories, solutions and instruments. Prefer using instruments when possible.
2. Check the online sources output of your knowledge_tool.
- Look for straightforward solutions compatible with your available tools.
- Always look for opensource python/nodejs/terminal tools and packages first.
diff --git a/prompts/default/agent.system.main.tips.md b/prompts/default/agent.system.main.tips.md
index 4b3dc567b..9298c6820 100644
--- a/prompts/default/agent.system.main.tips.md
+++ b/prompts/default/agent.system.main.tips.md
@@ -14,6 +14,10 @@
- Communication is the key to succesfull solution.
- NEVER delegate your whole task, only parts of it.
+## Instruments
+- Instruments are programs you can utilize to solve tasks
+- Instrument descriptions are injected into the prompt and can be executed with code_execution_tool
+
## Tips and tricks
- Focus on python/nodejs/linux libraries when searching for solutions. You can use them with your tools and make solutions easy.
- Sometimes you don't need tools, some things can be determined.
diff --git a/python/extensions/message_loop_prompts/_50_recall_memories.py b/python/extensions/message_loop_prompts/_50_recall_memories.py
index a3a1cb951..24fe706bb 100644
--- a/python/extensions/message_loop_prompts/_50_recall_memories.py
+++ b/python/extensions/message_loop_prompts/_50_recall_memories.py
@@ -60,12 +60,12 @@ class RecallMemories(Extension):
# log the short result
if not isinstance(memories, list) or len(memories) == 0:
log_item.update(
- heading="No useful memories found.",
+ heading="No useful memories found",
)
return
else:
log_item.update(
- heading=f"\n\n{len(memories)} memories found.",
+ heading=f"{len(memories)} memories found",
)
# concatenate memory.page_content in memories:
diff --git a/python/extensions/message_loop_prompts/_51_recall_solutions.py b/python/extensions/message_loop_prompts/_51_recall_solutions.py
index d05c2500f..4b177a073 100644
--- a/python/extensions/message_loop_prompts/_51_recall_solutions.py
+++ b/python/extensions/message_loop_prompts/_51_recall_solutions.py
@@ -7,7 +7,8 @@ class RecallSolutions(Extension):
INTERVAL = 3
HISTORY = 5
- RESULTS = 3
+ SOLUTIONS_COUNT = 2
+ INSTRUMENTS_COUNT = 2
THRESHOLD = 0.1
async def execute(self, loop_data: LoopData = LoopData(), **kwargs):
@@ -19,74 +20,78 @@ class RecallSolutions(Extension):
async def search_solutions(self, loop_data: LoopData, **kwargs):
# try:
- # show temp info message
- self.agent.context.log.log(
- type="info", content="Searching memory for solutions...", temp=True
+ # show temp info message
+ self.agent.context.log.log(
+ type="info", content="Searching memory for solutions...", temp=True
+ )
+
+ # show full util message, this will hide temp message immediately if turned on
+ log_item = self.agent.context.log.log(
+ type="util",
+ heading="Searching memory for solutions...",
+ )
+
+ # get system message and chat history for util llm
+ msgs_text = self.agent.concat_messages(
+ self.agent.history[-RecallSolutions.HISTORY :]
+ ) # only last X messages
+ system = self.agent.read_prompt(
+ "memory.solutions_query.sys.md", history=msgs_text
+ )
+
+ # log query streamed by LLM
+ def log_callback(content):
+ log_item.stream(query=content)
+
+ # call util llm to summarize conversation
+ query = await self.agent.call_utility_llm(
+ system=system, msg=loop_data.message, callback=log_callback
+ )
+
+ # get solutions database
+ db = await Memory.get(self.agent)
+
+ solutions = await db.search_similarity_threshold(
+ query=query,
+ limit=RecallSolutions.SOLUTIONS_COUNT,
+ threshold=RecallSolutions.THRESHOLD,
+ filter=f"area == '{Memory.Area.SOLUTIONS.value}'",
+ )
+ instruments = await db.search_similarity_threshold(
+ query=query,
+ limit=RecallSolutions.INSTRUMENTS_COUNT,
+ threshold=RecallSolutions.THRESHOLD,
+ filter=f"area == '{Memory.Area.INSTRUMENTS.value}'",
+ )
+
+ log_item.update(
+ heading=f"{len(instruments)} instruments, {len(solutions)} solutions found",
+ )
+
+ if instruments:
+ instruments_text = ""
+ for instrument in instruments:
+ instruments_text += instrument.page_content + "\n\n"
+ instruments_text = instruments_text.strip()
+ log_item.update(instruments=instruments_text)
+ instruments_prompt = self.agent.read_prompt(
+ "agent.system.instruments.md", instruments=instruments_text
)
+ loop_data.system.append(instruments_prompt)
- # show full util message, this will hide temp message immediately if turned on
- log_item = self.agent.context.log.log(
- type="util",
- heading="Searching memory for solutions...",
- )
-
- # get system message and chat history for util llm
- msgs_text = self.agent.concat_messages(
- self.agent.history[-RecallSolutions.HISTORY :]
- ) # only last X messages
- system = self.agent.read_prompt(
- "memory.solutions_query.sys.md", history=msgs_text
- )
-
- # log query streamed by LLM
- def log_callback(content):
- log_item.stream(query=content)
-
- # call util llm to summarize conversation
- query = await self.agent.call_utility_llm(
- system=system, msg=loop_data.message, callback=log_callback
- )
-
- # get solutions database
- db = await Memory.get(self.agent)
-
- solutions = await db.search_similarity_threshold(
- query=query,
- limit=RecallSolutions.RESULTS,
- threshold=RecallSolutions.THRESHOLD,
- filter=f"area == '{Memory.Area.SOLUTIONS.value}'"
- )
-
- # log the short result
- if not isinstance(solutions, list) or len(solutions) == 0:
- log_item.update(
- heading="No successful solution memories found.",
- )
- return
- else:
- log_item.update(
- heading=f"\n\n{len(solutions)} successful solution memories found.",
- )
-
- # concatenate solution.page_content in solutions:
+ if solutions:
solutions_text = ""
for solution in solutions:
solutions_text += solution.page_content + "\n\n"
solutions_text = solutions_text.strip()
-
- # log the full results
log_item.update(solutions=solutions_text)
-
- # place to prompt
solutions_prompt = self.agent.read_prompt(
"agent.system.solutions.md", solutions=solutions_text
)
-
- # append to system message
loop_data.system.append(solutions_prompt)
- # except Exception as e:
- # err = errors.format_error(e)
- # self.agent.context.log.log(
- # type="error", heading="Recall solutions extension error:", content=err
- # )
+ # except Exception as e:
+ # err = errors.format_error(e)
+ # self.agent.context.log.log(
+ # type="error", heading="Recall solutions extension error:", content=err
+ # )
diff --git a/python/helpers/knowledge_import.py b/python/helpers/knowledge_import.py
index 7c2e65d0d..7e3b7a8c1 100644
--- a/python/helpers/knowledge_import.py
+++ b/python/helpers/knowledge_import.py
@@ -34,11 +34,14 @@ def calculate_checksum(file_path: str) -> str:
def load_knowledge(
- log_item: LogItem | None, knowledge_dir: str, index: Dict[str, KnowledgeImport]
+ log_item: LogItem | None,
+ knowledge_dir: str,
+ index: Dict[str, KnowledgeImport],
+ metadata: dict[str, Any] = {},
+ filename_pattern: str = "**/*",
) -> Dict[str, KnowledgeImport]:
- knowledge_dir = files.get_abs_path("knowledge",knowledge_dir)
- from python.helpers.memory import Memory
+ # from python.helpers.memory import Memory
# Mapping file extensions to corresponding loader classes
file_types_loaders = {
@@ -47,62 +50,67 @@ def load_knowledge(
"csv": CSVLoader,
"html": UnstructuredHTMLLoader,
"json": JSONLoader,
- "md": UnstructuredMarkdownLoader,
+ # "md": UnstructuredMarkdownLoader,
+ "md": TextLoader,
}
cnt_files = 0
cnt_docs = 0
- for area in Memory.Area:
- subdir = files.get_abs_path(knowledge_dir, area.value)
+ # for area in Memory.Area:
+ # subdir = files.get_abs_path(knowledge_dir, area.value)
- if not os.path.exists(subdir):
- os.makedirs(subdir)
- continue
+ # if not os.path.exists(knowledge_dir):
+ # os.makedirs(knowledge_dir)
+ # continue
- # Fetch all files in the directory with specified extensions
- kn_files = glob.glob(subdir + "/**/*", recursive=True)
- if kn_files:
- print(f"Found {len(kn_files)} knowledge files in {subdir}, processing...")
- if log_item:
- log_item.stream(
- progress=f"\nFound {len(kn_files)} knowledge files in {subdir}, processing...",
- )
+ # Fetch all files in the directory with specified extensions
+ kn_files = glob.glob(knowledge_dir + "/" + filename_pattern, recursive=True)
+ kn_files = [f for f in kn_files if os.path.isfile(f)]
- for file_path in kn_files:
- ext = file_path.split(".")[-1].lower()
- if ext in file_types_loaders:
- checksum = calculate_checksum(file_path)
- file_key = file_path # os.path.relpath(file_path, knowledge_dir)
-
- # Load existing data from the index or create a new entry
- file_data = index.get(file_key, {})
-
- if file_data.get("checksum") == checksum:
- file_data["state"] = "original"
- else:
- file_data["state"] = "changed"
-
- if file_data["state"] == "changed":
- file_data["checksum"] = checksum
- loader_cls = file_types_loaders[ext]
- loader = loader_cls(
- file_path,
- **(
- text_loader_kwargs
- if ext in ["txt", "csv", "html", "md"]
- else {}
- ),
- )
- file_data["documents"] = loader.load_and_split()
- for doc in file_data["documents"]:
- doc.metadata["area"] = area.value
- cnt_files += 1
- cnt_docs += len(file_data["documents"])
- # print(f"Imported {len(file_data['documents'])} documents from {file_path}")
-
- # Update the index
- index[file_key] = file_data # type: ignore
+ if kn_files:
+ print(
+ f"Found {len(kn_files)} knowledge files in {knowledge_dir}, processing..."
+ )
+ if log_item:
+ log_item.stream(
+ progress=f"\nFound {len(kn_files)} knowledge files in {knowledge_dir}, processing...",
+ )
+
+ for file_path in kn_files:
+ ext = file_path.split(".")[-1].lower()
+ if ext in file_types_loaders:
+ checksum = calculate_checksum(file_path)
+ file_key = file_path # os.path.relpath(file_path, knowledge_dir)
+
+ # Load existing data from the index or create a new entry
+ file_data = index.get(file_key, {})
+
+ if file_data.get("checksum") == checksum:
+ file_data["state"] = "original"
+ else:
+ file_data["state"] = "changed"
+
+ if file_data["state"] == "changed":
+ file_data["checksum"] = checksum
+ loader_cls = file_types_loaders[ext]
+ loader = loader_cls(
+ file_path,
+ **(
+ text_loader_kwargs
+ if ext in ["txt", "csv", "html", "md"]
+ else {}
+ ),
+ )
+ file_data["documents"] = loader.load_and_split()
+ for doc in file_data["documents"]:
+ doc.metadata = {**doc.metadata, **metadata}
+ cnt_files += 1
+ cnt_docs += len(file_data["documents"])
+ # print(f"Imported {len(file_data['documents'])} documents from {file_path}")
+
+ # Update the index
+ index[file_key] = file_data # type: ignore
# loop index where state is not set and mark it as removed
for file_key, file_data in index.items():
diff --git a/python/helpers/memory.py b/python/helpers/memory.py
index 417ef4790..fb14508e0 100644
--- a/python/helpers/memory.py
+++ b/python/helpers/memory.py
@@ -21,20 +21,23 @@ from python.helpers.log import Log, LogItem
from enum import Enum
from agent import Agent
+
class MyFaiss(FAISS):
- #override aget_by_ids
+ # override aget_by_ids
def get_by_ids(self, ids: Sequence[str], /) -> List[Document]:
# return all self.docstore._dict[id] in ids
- return [self.docstore._dict[id] for id in ids if id in self.docstore._dict] #type: ignore
+ return [self.docstore._dict[id] for id in ids if id in self.docstore._dict] # type: ignore
async def aget_by_ids(self, ids: Sequence[str], /) -> List[Document]:
return self.get_by_ids(ids)
+
class Memory:
class Area(Enum):
MAIN = "main"
SOLUTIONS = "solutions"
+ INSTRUMENTS = "instruments"
index: dict[str, "MyFaiss"] = {}
@@ -130,7 +133,7 @@ class Memory:
# normalize_L2=True,
relevance_score_fn=Memory._cosine_normalizer,
)
- return db # type: ignore
+ return db # type: ignore
def __init__(
self,
@@ -160,8 +163,8 @@ class Memory:
with open(index_path, "r") as f:
index = json.load(f)
- for kn_dir in kn_dirs:
- index = knowledge_import.load_knowledge(log_item, kn_dir, index)
+ # preload knowledge folders
+ index = self._preload_knowledge_folders(log_item, kn_dirs, index)
for file in index:
if index[file]["state"] in ["changed", "removed"] and index[file].get(
@@ -187,6 +190,33 @@ class Memory:
with open(index_path, "w") as f:
json.dump(index, f)
+ def _preload_knowledge_folders(
+ self,
+ log_item: LogItem | None,
+ kn_dirs: list[str],
+ index: dict[str, knowledge_import.KnowledgeImport],
+ ):
+ # load knowledge folders, subfolders by area
+ for kn_dir in kn_dirs:
+ for area in Memory.Area:
+ index = knowledge_import.load_knowledge(
+ log_item,
+ files.get_abs_path("knowledge", kn_dir, area.value),
+ index,
+ {"area": area.value},
+ )
+
+ # load instruments descriptions
+ index = knowledge_import.load_knowledge(
+ log_item,
+ files.get_abs_path("instruments"),
+ index,
+ {"area": Memory.Area.INSTRUMENTS.value},
+ filename_pattern="**/*.md",
+ )
+
+ return index
+
async def search_similarity_threshold(
self, query: str, limit: int, threshold: float, filter: str = ""
):
@@ -235,9 +265,9 @@ class Memory:
async def delete_documents_by_ids(self, ids: list[str]):
# aget_by_ids is not yet implemented in faiss, need to do a workaround
- rem_docs =self.db.get_by_ids(ids) # existing docs to remove (prevents error)
+ rem_docs = self.db.get_by_ids(ids) # existing docs to remove (prevents error)
if rem_docs:
- rem_ids = [doc.metadata["id"] for doc in rem_docs] # ids to remove
+ rem_ids = [doc.metadata["id"] for doc in rem_docs] # ids to remove
await self.db.adelete(ids=rem_ids)
if rem_docs:
diff --git a/python/tools/code_execution_tool.py b/python/tools/code_execution_tool.py
index eae0a7ba6..78c071be1 100644
--- a/python/tools/code_execution_tool.py
+++ b/python/tools/code_execution_tool.py
@@ -36,7 +36,7 @@ class CodeExecution(Tool):
response = await self.execute_terminal_command(self.args["code"])
elif runtime == "output":
response = await self.get_terminal_output(
- wait_with_output=5, wait_without_output=20
+ wait_with_output=5, wait_without_output=60
)
elif runtime == "reset":
response = await self.reset_terminal()
@@ -137,7 +137,7 @@ class CodeExecution(Tool):
return await self.get_terminal_output()
async def get_terminal_output(
- self, wait_with_output=3, wait_without_output=10, max_exec_time=15
+ self, wait_with_output=3, wait_without_output=10, max_exec_time=60
):
idle = 0
SLEEP_TIME = 0.1