Agent Zero AI framework
Squashed commit of the following:
frdel9c7339042f1c
diff --git a/.gitignore b/.gitignoreindex e10c1fd6b..5a668772b 100644--- a/.gitignore+++ b/.gitignore@@ -65,4 +65,4 @@ instruments/**/*.*# Ignore all contents of the directory "bin"bin/*# But do not ignore the directory itself-!bin/.gitkeep\ No newline at end of file+!bin/.gitkeepdiff --git a/.vscode/launch.json b/.vscode/launch.jsonindex 37ab9f096..883a8edbc 100644--- a/.vscode/launch.json+++ b/.vscode/launch.json@@ -1,29 +1,29 @@{- "version": "0.2.0",- "configurations": [- {- "name": "Debug run_ui.py",- "type": "debugpy",- "request": "launch",- "program": "./run_ui.py",- "console": "integratedTerminal",- "args": ["-Xfrozen_modules=off"]- },- {- "name": "Debug run_cli.py",- "type": "debugpy",- "request": "launch",- "program": "./run_cli.py",- "console": "integratedTerminal",- "args": ["-Xfrozen_modules=off"]- },- {- "name": "Debug current file",- "type": "debugpy",- "request": "launch",- "program": "${file}",- "console": "integratedTerminal",- "args": ["-Xfrozen_modules=off"]- }- ]-}\ No newline at end of file+ "version": "0.2.0",+ "configurations": [+ {+ "name": "Debug run_ui.py",+ "type": "debugpy",+ "request": "launch",+ "program": "./run_ui.py",+ "console": "integratedTerminal",+ "args": ["--development=true", "-Xfrozen_modules=off"]+ },+ {+ "name": "Debug run_cli.py",+ "type": "debugpy",+ "request": "launch",+ "program": "./run_cli.py",+ "console": "integratedTerminal",+ "args": ["--development=true", "-Xfrozen_modules=off"]+ },+ {+ "name": "Debug current file",+ "type": "debugpy",+ "request": "launch",+ "program": "${file}",+ "console": "integratedTerminal",+ "args": ["--development=true", "-Xfrozen_modules=off"]+ }+ ]+}diff --git a/agent.py b/agent.pyindex 2bac75793..f3d8a4f60 100644--- a/agent.py+++ b/agent.py@@ -457,7 +457,7 @@ class Agent:printer = PrintStyle(italic=True, font_color="orange", padding=False)def log_callback(content):- printer.print(content)+ printer.stream(content)log_item.stream(content=content)summary = await self.call_utility_llm(diff --git a/docker/docker_manager_macos.sh b/docker/docker_manager_mac_linux.shsimilarity index 100%rename from docker/docker_manager_macos.shrename to docker/docker_manager_mac_linux.shdiff --git a/docker/run/Dockerfile b/docker/run/Dockerfileindex 5049a5d3e..c38b778c6 100644--- a/docker/run/Dockerfile+++ b/docker/run/Dockerfile@@ -21,9 +21,6 @@ RUN apt-get update && apt-get install -y \# Cleanup package listRUN rm -rf /var/lib/apt/lists/*-# Clone Agent Zero repository-RUN git clone --branch development https://github.com/frdel/agent-zero.git /git/agent-zero-# Set up SSHRUN mkdir /var/run/sshd && \echo 'root:toor' | chpasswd && \@@ -33,12 +30,8 @@ RUN mkdir /var/run/sshd && \ENV VIRTUAL_ENV=/opt/venvRUN python3 -m venv $VIRTUAL_ENV-# Copy the script to ensure .bashrc is in the root directory-COPY initialize.sh /usr/local/bin/initialize.sh-RUN chmod +x /usr/local/bin/initialize.sh--# Copy contents of filesystem directory to /fs-COPY ./fs/ /fs+# Copy contents of filesystem directory to /+COPY ./fs/ /# Ensure the virtual environment and pip setupRUN $VIRTUAL_ENV/bin/pip install --upgrade pip@@ -48,13 +41,27 @@ RUN $VIRTUAL_ENV/bin/pip install \ipython \requests+# Clone Agent Zero repository+RUN git clone --branch development https://github.com/frdel/agent-zero.git /git/agent-zero+# Install A0 python packagesRUN $VIRTUAL_ENV/bin/pip install -r /git/agent-zero/requirements.txt# Preload A0RUN $VIRTUAL_ENV/bin/python /git/agent-zero/preload.py+++# Clone additional git repos+RUN git clone https://github.com/searxng/searxng.git /git/searxng+++# install additional software+RUN bash /ins/install_searxng.sh+++# Expose portsEXPOSE 22 80-# Init .bashrc-CMD ["/usr/local/bin/initialize.sh"]\ No newline at end of file+# initialize runtime+CMD ["/bin/bash", "/exe/initialize.sh"]\ No newline at end of filediff --git a/docker/run/build.txt b/docker/run/build.txtindex 605762818..d5d52dd2d 100644--- a/docker/run/build.txt+++ b/docker/run/build.txt@@ -3,4 +3,4 @@ docker build -t agent-zero-run:latest .# dockerhubdocker login-docker buildx build --platform linux/amd64,linux/arm64 -t frdel/agent-zero-run:latest --push .\ No newline at end of file+docker buildx build --platform linux/amd64,linux/arm64 -t frdel/agent-zero-run:testing --push .\ No newline at end of filediff --git a/docker/run/fs/etc/searxng/settings.yml b/docker/run/fs/etc/searxng/settings.ymlnew file mode 100644index 000000000..7b9f3b0d1--- /dev/null+++ b/docker/run/fs/etc/searxng/settings.yml@@ -0,0 +1,77 @@+# SearXNG settings++use_default_settings: true++general:+ debug: false+ instance_name: "SearXNG"++search:+ safe_search: 2+ autocomplete: 'duckduckgo'+ formats:+ - html+ - json++server:+ # Is overwritten by ${SEARXNG_SECRET}+ secret_key: "1e91dce279fdef87eb7b6f43029c206b"+ limiter: true+ image_proxy: true+ # public URL of the instance, to ensure correct inbound links. Is overwritten+ # by ${SEARXNG_URL}.+ # base_url: http://example.com/location++redis:+ # URL to connect redis database. Is overwritten by ${SEARXNG_REDIS_URL}.+ url: unix:///usr/local/searxng-redis/run/redis.sock?db=0++ui:+ static_use_hash: true++# preferences:+# lock:+# - autocomplete+# - method++enabled_plugins:+ - 'Hash plugin'+ - 'Self Informations'+ - 'Tracker URL remover'+ - 'Ahmia blacklist'+ # - 'Hostnames plugin' # see 'hostnames' configuration below+ # - 'Open Access DOI rewrite'++# plugins:+# - only_show_green_results++# hostnames:+# replace:+# '(.*\.)?youtube\.com$': 'invidious.example.com'+# '(.*\.)?youtu\.be$': 'invidious.example.com'+# remove:+# - '(.*\.)?facebook.com$'+# low_priority:+# - '(.*\.)?google\.com$'+# high_priority:+# - '(.*\.)?wikipedia.org$'++engines:++# - name: fdroid+# disabled: false+#+# - name: apk mirror+# disabled: false+#+# - name: mediathekviewweb+# categories: TV+# disabled: false+#+# - name: invidious+# disabled: false+# base_url:+# - https://invidious.snopyta.org+# - https://invidious.tiekoetter.com+# - https://invidio.xamh.de+# - https://inv.riverside.rocks\ No newline at end of filediff --git a/docker/run/initialize.sh b/docker/run/fs/exe/initialize.shsimilarity index 61%rename from docker/run/initialize.shrename to docker/run/fs/exe/initialize.shindex 1f9b0db5b..993379053 100644--- a/docker/run/initialize.sh+++ b/docker/run/fs/exe/initialize.sh@@ -1,18 +1,21 @@#!/bin/bash-# Copy all contents from /fs to root directory (/) without overwriting-cp -rn --no-preserve=ownership,mode /fs/* /+# Copy all contents from persistent /per to root directory (/) without overwriting+cp -r --no-preserve=ownership,mode /per/* /# allow execution of /root/.bashrc and /root/.profilechmod 444 /root/.bashrcchmod 444 /root/.profile# update package list to save time later-apt-get update+apt-get update &# Start SSH service in background/usr/sbin/sshd -D &+# Start searxng server in background+sudo -H -u searxng -i bash /exe/run_searxng.sh &+# Start A0 and restart on exitbash /exe/run_A0.shif [ $? -ne 0 ]; thendiff --git a/docker/run/fs/exe/run_A0.sh b/docker/run/fs/exe/run_A0.shindex 266c6459c..0da64a97f 100644--- a/docker/run/fs/exe/run_A0.sh+++ b/docker/run/fs/exe/run_A0.sh@@ -34,14 +34,14 @@ while true; doecho "Starting A0..."python /a0/run_ui.py \- --port 80 \- --host "0.0.0.0" \- --code_exec_docker_enabled False \- --code_exec_ssh_enabled True \- --code_exec_ssh_addr "localhost" \- --code_exec_ssh_port 22 \- --code_exec_ssh_user "root" \- --code_exec_ssh_pass "toor"+ --port=80 \+ --host="0.0.0.0" \+ --code_exec_docker_enabled=false \+ --code_exec_ssh_enabled=true \+ --code_exec_ssh_addr="localhost" \+ --code_exec_ssh_port=22 \+ --code_exec_ssh_user="root" \+ --code_exec_ssh_pass="toor"# Check the exit statusif [ $? -ne 0 ]; thendiff --git a/docker/run/fs/exe/run_searxng.sh b/docker/run/fs/exe/run_searxng.shnew file mode 100644index 000000000..880314f29--- /dev/null+++ b/docker/run/fs/exe/run_searxng.sh@@ -0,0 +1,7 @@+#!/bin/bash++# start webapp+sudo -H -u searxng -i+cd /usr/local/searxng/searxng-src+export SEARXNG_SETTINGS_PATH="/etc/searxng/settings.yml"+python searx/webapp.py\ No newline at end of filediff --git a/docker/run/fs/ins/install_searxng.sh b/docker/run/fs/ins/install_searxng.shnew file mode 100644index 000000000..ac76a30da--- /dev/null+++ b/docker/run/fs/ins/install_searxng.sh@@ -0,0 +1,17 @@+#!/bin/bash++sudo -H apt-get install -y \+ python3-dev python3-babel python3-venv \+ uwsgi uwsgi-plugin-python3 \+ git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev++sudo -H useradd --shell /bin/bash --system \+ --home-dir "/usr/local/searxng" \+ --comment 'Privacy-respecting metasearch engine' \+ searxng++sudo -H mkdir "/usr/local/searxng"+sudo -H chown -R "searxng:searxng" "/usr/local/searxng"++# Start a new shell from new created user and clone SearXNG:+sudo -H -u searxng -i bash /ins/install_searxng_user.shdiff --git a/docker/run/fs/ins/install_searxng_user.sh b/docker/run/fs/ins/install_searxng_user.shnew file mode 100644index 000000000..0bc2132bf--- /dev/null+++ b/docker/run/fs/ins/install_searxng_user.sh@@ -0,0 +1,26 @@+#!/bin/bash++# clone SearXNG repo+# git clone "https://github.com/searxng/searxng" \+# "/usr/local/searxng/searxng-src"+cp -r "/git/searxng" "/usr/local/searxng/searxng-src"++# create virtualenv:+python3 -m venv "/usr/local/searxng/searx-pyenv"++# make it default+echo ". /usr/local/searxng/searx-pyenv/bin/activate" \+ >> "/usr/local/searxng/.profile"++# activate venv+source "/usr/local/searxng/searx-pyenv/bin/activate"++# update pip's boilerplate+pip install -U pip+pip install -U setuptools+pip install -U wheel+pip install -U pyyaml++# jump to SearXNG's working tree and install SearXNG into virtualenv+cd "/usr/local/searxng/searxng-src"+pip install --use-pep517 --no-build-isolation -e .diff --git a/docker/run/fs/root/.bashrc b/docker/run/fs/per/root/.bashrcsimilarity index 100%rename from docker/run/fs/root/.bashrcrename to docker/run/fs/per/root/.bashrcdiff --git a/docker/run/fs/root/.profile b/docker/run/fs/per/root/.profilesimilarity index 100%rename from docker/run/fs/root/.profilerename to docker/run/fs/per/root/.profilediff --git a/initialize.py b/initialize.pyindex e8b6d8611..24e84aa67 100644--- a/initialize.py+++ b/initialize.py@@ -1,15 +1,12 @@import modelsfrom agent import AgentConfig-from python.helpers import files, settings+from python.helpers import files, runtime, settings-global_kwargs = {}--def set_global_kwargs(**kwargs):- global global_kwargs- global_kwargs = kwargsdef initialize():-++ current_settings = settings.get_settings()+# main chat model used by agents (smarter, more accurate)# chat_llm = models.get_openai_chat(model_name="gpt-4o-mini", temperature=0)# chat_llm = models.get_ollama_chat(model_name="llama3.2:3b-instruct-fp16", temperature=0)@@ -21,44 +18,50 @@ def initialize():# chat_llm = models.get_mistral_chat(model_name="mistral-small-latest", temperature=0)# chat_llm = models.get_groq_chat(model_name="llama-3.2-90b-text-preview", temperature=0)# chat_llm = models.get_sambanova_chat(model_name="Meta-Llama-3.1-70B-Instruct-8k", temperature=0)- chat_llm = settings.get_chat_model() # chat model from user settings+ chat_llm = settings.get_chat_model(+ current_settings+ ) # chat model from user settings# utility model used for helper functions (cheaper, faster)# utility_llm = chat_llm- utility_llm = settings.get_utility_model() # utility model from user settings+ utility_llm = settings.get_utility_model(+ current_settings+ ) # utility model from user settings# embedding model used for memory# embedding_llm = models.get_openai_embedding(model_name="text-embedding-3-small")# embedding_llm = models.get_ollama_embedding(model_name="nomic-embed-text")# embedding_llm = models.get_huggingface_embedding(model_name="sentence-transformers/all-MiniLM-L6-v2")# embedding_llm = models.get_lmstudio_embedding(model_name="nomic-ai/nomic-embed-text-v1.5-GGUF")- embedding_llm = settings.get_embedding_model() # embedding model from user settings-+ embedding_llm = settings.get_embedding_model(+ current_settings+ ) # embedding model from user settings+# agent configurationconfig = AgentConfig(- chat_model = chat_llm,- utility_model = utility_llm,- embeddings_model = embedding_llm,- # prompts_subdir = "default",- # memory_subdir = "",- knowledge_subdirs = ["default","custom"],+ chat_model=chat_llm,+ utility_model=utility_llm,+ embeddings_model=embedding_llm,+ prompts_subdir=current_settings["agent_prompts_subdir"],+ memory_subdir=current_settings["agent_memory_subdir"],+ knowledge_subdirs=["default", current_settings["agent_knowledge_subdir"]],# rate_limit_seconds = 60,- rate_limit_requests = 30,+ rate_limit_requests=30,# rate_limit_input_tokens = 0,# rate_limit_output_tokens = 0,# msgs_keep_max = 25,# msgs_keep_start = 5,# msgs_keep_end = 10,- max_tool_response_length = 3000,+ max_tool_response_length=3000,# response_timeout_seconds = 60,# code_exec_docker_enabled = True,# code_exec_docker_name = "agent-zero-exe",# code_exec_docker_image = "frdel/agent-zero-exe:latest",# code_exec_docker_ports = { "22/tcp": 50022 }- # code_exec_docker_volumes = {- # files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"},- # files.get_abs_path("instruments"): {"bind": "/instruments", "mode": "rw"},- # },+ # code_exec_docker_volumes = {+ # files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"},+ # files.get_abs_path("instruments"): {"bind": "/instruments", "mode": "rw"},+ # },# code_exec_ssh_enabled = True,# code_exec_ssh_addr = "localhost",# code_exec_ssh_port = 50022,@@ -68,9 +71,27 @@ def initialize():)# update config with kwargs- for key, value in global_kwargs.items():+ for key, value in runtime.args.items():if hasattr(config, key):+ # conversion based on type of config[key]+ if isinstance(getattr(config, key), bool):+ value = value.lower().strip() == "true"+ print("bool", value)+ elif isinstance(getattr(config, key), int):+ value = int(value)+ print("int", value)+ elif isinstance(getattr(config, key), float):+ value = float(value)+ print("float", value)+ elif isinstance(getattr(config, key), str):+ value = str(value)+ print("str", value)+ else:+ raise Exception(+ f"Unsupported argument type of '{key}': {type(getattr(config, key))}"+ )+setattr(config, key, value)-+# return config objectreturn configdiff --git a/instruments/default/yt_download/yt_download.md b/instruments/default/yt_download/yt_download.mdindex 25f09ffd6..0316d0583 100644--- a/instruments/default/yt_download/yt_download.md+++ b/instruments/default/yt_download/yt_download.md@@ -2,5 +2,5 @@Download a YouTube video# Solution1. If folder is specified, cd to it-2. Run instrument "bash /instruments/default/yt_download/yt_download.sh <url>" with your video URL+2. Run instrument "bash /a0/instruments/default/yt_download/yt_download.sh <url>" with your video URL3. Wait for the terminal to finish\ No newline at end of filediff --git a/models.py b/models.pyindex ed194bae3..85b0805db 100644--- a/models.py+++ b/models.py@@ -58,7 +58,11 @@ class ModelProvider(Enum):# Utility function to get API keys from environment variablesdef get_api_key(service):- return dotenv.get_dotenv_value(f"API_KEY_{service.upper()}") or dotenv.get_dotenv_value(f"{service.upper()}_API_KEY") or "None"+ return (+ dotenv.get_dotenv_value(f"API_KEY_{service.upper()}")+ or dotenv.get_dotenv_value(f"{service.upper()}_API_KEY")+ or "None"+ )def get_model(type: ModelType, provider: ModelProvider, name: str, **kwargs):@@ -71,10 +75,12 @@ def get_model(type: ModelType, provider: ModelProvider, name: str, **kwargs):def get_ollama_chat(model_name: str,temperature=DEFAULT_TEMPERATURE,- base_url=dotenv.get_dotenv_value("OLLAMA_BASE_URL") or "http://127.0.0.1:11434",+ base_url=None,num_ctx=8192,**kwargs,):+ if not base_url:+ base_url = dotenv.get_dotenv_value("OLLAMA_BASE_URL") or "http://127.0.0.1:11434"return ChatOllama(model=model_name,temperature=temperature,@@ -87,9 +93,11 @@ def get_ollama_chat(def get_ollama_embedding(model_name: str,temperature=DEFAULT_TEMPERATURE,- base_url=dotenv.get_dotenv_value("OLLAMA_BASE_URL") or "http://127.0.0.1:11434",+ base_url=None,**kwargs,):+ if not base_url:+ base_url = dotenv.get_dotenv_value("OLLAMA_BASE_URL") or "http://127.0.0.1:11434"return OllamaEmbeddings(model=model_name, temperature=temperature, base_url=base_url, **kwargs)@@ -98,13 +106,13 @@ def get_ollama_embedding(# HuggingFace modelsdef get_huggingface_chat(model_name: str,- api_key=get_api_key("huggingface"),+ api_key=None,temperature=DEFAULT_TEMPERATURE,**kwargs,):# different naming convention hereif not api_key:- api_key = os.environ["HUGGINGFACEHUB_API_TOKEN"]+ api_key = get_api_key("huggingface") or os.environ["HUGGINGFACEHUB_API_TOKEN"]# Initialize the HuggingFaceEndpoint with the specified model and parametersllm = HuggingFaceEndpoint(@@ -127,7 +135,8 @@ def get_huggingface_embedding(model_name: str, **kwargs):def get_lmstudio_chat(model_name: str,temperature=DEFAULT_TEMPERATURE,- base_url=dotenv.get_dotenv_value("LM_STUDIO_BASE_URL") or "http://127.0.0.1:1234/v1",+ base_url=dotenv.get_dotenv_value("LM_STUDIO_BASE_URL")+ or "http://127.0.0.1:1234/v1",**kwargs,):return ChatOpenAI(model_name=model_name, base_url=base_url, temperature=temperature, api_key="none", **kwargs) # type: ignore@@ -135,7 +144,8 @@ def get_lmstudio_chat(def get_lmstudio_embedding(model_name: str,- base_url=dotenv.get_dotenv_value("LM_STUDIO_BASE_URL") or "http://127.0.0.1:1234/v1",+ base_url=dotenv.get_dotenv_value("LM_STUDIO_BASE_URL")+ or "http://127.0.0.1:1234/v1",**kwargs,):return OpenAIEmbeddings(model=model_name, api_key="none", base_url=base_url, check_embedding_ctx_length=False, **kwargs) # type: ignore@@ -144,151 +154,172 @@ def get_lmstudio_embedding(# Anthropic modelsdef get_anthropic_chat(model_name: str,- api_key=get_api_key("anthropic"),+ api_key=None,temperature=DEFAULT_TEMPERATURE,**kwargs,):+ if not api_key:+ api_key = get_api_key("anthropic")return ChatAnthropic(model_name=model_name, temperature=temperature, api_key=api_key, **kwargs) # type: ignore# right now anthropic does not have embedding models, but that might changedef get_anthropic_embedding(model_name: str,- api_key=get_api_key("anthropic"),+ api_key=None,**kwargs,):+ if not api_key:+ api_key = get_api_key("anthropic")return OpenAIEmbeddings(model=model_name, api_key=api_key, **kwargs) # type: ignore# OpenAI modelsdef get_openai_chat(model_name: str,- api_key=get_api_key("openai"),+ api_key=None,temperature=DEFAULT_TEMPERATURE,**kwargs,):+ if not api_key:+ api_key = get_api_key("openai")return ChatOpenAI(model_name=model_name, temperature=temperature, api_key=api_key, **kwargs) # type: ignore-def get_openai_instruct(- model_name: str,- api_key=get_api_key("openai"),- temperature=DEFAULT_TEMPERATURE,- **kwargs,-):- return OpenAI(model=model_name, temperature=temperature, api_key=api_key, **kwargs) # type: ignore---def get_openai_embedding(model_name: str, api_key=get_api_key("openai"), **kwargs):+def get_openai_embedding(model_name: str, api_key=None, **kwargs):+ if not api_key:+ api_key = get_api_key("openai")return OpenAIEmbeddings(model=model_name, api_key=api_key, **kwargs) # type: ignoredef get_azure_openai_chat(deployment_name: str,- api_key=get_api_key("openai_azure"),+ api_key=None,temperature=DEFAULT_TEMPERATURE,- azure_endpoint=dotenv.get_dotenv_value("OPENAI_AZURE_ENDPOINT"),+ azure_endpoint=None,**kwargs,):+ if not api_key:+ api_key = get_api_key("openai_azure")+ if not azure_endpoint:+ azure_endpoint = dotenv.get_dotenv_value("OPENAI_AZURE_ENDPOINT")return AzureChatOpenAI(deployment_name=deployment_name, temperature=temperature, api_key=api_key, azure_endpoint=azure_endpoint, **kwargs) # type: ignore-def get_azure_openai_instruct(- deployment_name: str,- api_key=get_api_key("openai_azure"),- temperature=DEFAULT_TEMPERATURE,- azure_endpoint=dotenv.get_dotenv_value("OPENAI_AZURE_ENDPOINT"),- **kwargs,-):- return AzureOpenAI(deployment_name=deployment_name, temperature=temperature, api_key=api_key, azure_endpoint=azure_endpoint, **kwargs) # type: ignore--def get_azure_openai_embedding(deployment_name: str,- api_key=get_api_key("openai_azure"),- azure_endpoint=dotenv.get_dotenv_value("OPENAI_AZURE_ENDPOINT"),+ api_key=None,+ azure_endpoint=None,**kwargs,):+ if not api_key:+ api_key = get_api_key("openai_azure")+ if not azure_endpoint:+ azure_endpoint = dotenv.get_dotenv_value("OPENAI_AZURE_ENDPOINT")return AzureOpenAIEmbeddings(deployment_name=deployment_name, api_key=api_key, azure_endpoint=azure_endpoint, **kwargs) # type: ignore# Google modelsdef get_google_chat(model_name: str,- api_key=get_api_key("google"),+ api_key=None,temperature=DEFAULT_TEMPERATURE,**kwargs,):+ if not api_key:+ api_key = get_api_key("google")return GoogleGenerativeAI(model=model_name, temperature=temperature, google_api_key=api_key, safety_settings={HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE}, **kwargs) # type: ignoredef get_google_embedding(model_name: str,- api_key=get_api_key("google"),+ api_key=None,**kwargs,):+ if not api_key:+ api_key = get_api_key("google")return google_embeddings.GoogleGenerativeAIEmbeddings(model=model_name, api_key=api_key, **kwargs) # type: ignore# Mistral modelsdef get_mistral_chat(model_name: str,- api_key=get_api_key("mistral"),+ api_key=None,temperature=DEFAULT_TEMPERATURE,**kwargs,):+ if not api_key:+ api_key = get_api_key("mistral")return ChatMistralAI(model=model_name, temperature=temperature, api_key=api_key, **kwargs) # type: ignore# Groq modelsdef get_groq_chat(model_name: str,- api_key=get_api_key("groq"),+ api_key=None,temperature=DEFAULT_TEMPERATURE,**kwargs,):+ if not api_key:+ api_key = get_api_key("groq")return ChatGroq(model_name=model_name, temperature=temperature, api_key=api_key, **kwargs) # type: ignore# OpenRouter modelsdef get_openrouter_chat(model_name: str,- api_key=get_api_key("openrouter"),+ api_key=None,temperature=DEFAULT_TEMPERATURE,- base_url=dotenv.get_dotenv_value("OPEN_ROUTER_BASE_URL") or "https://openrouter.ai/api/v1",+ base_url=None,**kwargs,):+ if not api_key:+ api_key = get_api_key("openrouter")+ if not base_url:+ base_url = dotenv.get_dotenv_value("OPEN_ROUTER_BASE_URL") or "https://openrouter.ai/api/v1"return ChatOpenAI(api_key=api_key, model=model_name, temperature=temperature, base_url=base_url, **kwargs) # type: ignoredef get_openrouter_embedding(model_name: str,- api_key=get_api_key("openrouter"),- base_url=dotenv.get_dotenv_value("OPEN_ROUTER_BASE_URL") or "https://openrouter.ai/api/v1",+ api_key=None,+ base_url=None,**kwargs,):+ if not api_key:+ api_key = get_api_key("openrouter")+ if not base_url:+ base_url = dotenv.get_dotenv_value("OPEN_ROUTER_BASE_URL") or "https://openrouter.ai/api/v1"return OpenAIEmbeddings(model=model_name, api_key=api_key, base_url=base_url, **kwargs) # type: ignore# Sambanova modelsdef get_sambanova_chat(model_name: str,- api_key=get_api_key("sambanova"),+ api_key=None,temperature=DEFAULT_TEMPERATURE,- base_url=dotenv.get_dotenv_value("SAMBANOVA_BASE_URL") or "https://fast-api.snova.ai/v1",+ base_url=None,max_tokens=1024,**kwargs,):+ if not api_key:+ api_key = get_api_key("sambanova")+ if not base_url:+ base_url = dotenv.get_dotenv_value("SAMBANOVA_BASE_URL") or "https://fast-api.snova.ai/v1"return ChatOpenAI(api_key=api_key, model=model_name, temperature=temperature, base_url=base_url, max_tokens=max_tokens, **kwargs) # type: ignore# right now sambanova does not have embedding models, but that might changedef get_sambanova_embedding(model_name: str,- api_key=get_api_key("sambanova"),- base_url=dotenv.get_dotenv_value("SAMBANOVA_BASE_URL") or "https://fast-api.snova.ai/v1",+ api_key=None,+ base_url=None,**kwargs,):+ if not api_key:+ api_key = get_api_key("sambanova")+ if not base_url:+ base_url = dotenv.get_dotenv_value("SAMBANOVA_BASE_URL") or "https://fast-api.snova.ai/v1"return OpenAIEmbeddings(model=model_name, api_key=api_key, base_url=base_url, **kwargs) # type: ignorediff --git a/preload.py b/preload.pyindex 85c75d76c..f820d5cab 100644--- a/preload.py+++ b/preload.py@@ -1,3 +1,3 @@-from python.helpers import whisper_oai+from python.helpers import whisper-whisper_oai.preload() # preload transcription model\ No newline at end of file+whisper.preload() # preload transcription model\ No newline at end of filediff --git a/prompts/compressed/agent.system.main.communication.md b/prompts/compressed/agent.system.main.communication.mdnew file mode 100644index 000000000..78d7504ca--- /dev/null+++ b/prompts/compressed/agent.system.main.communication.md@@ -0,0 +1,25 @@++## Communication+respond valid json with fields+thoughts: array thoughts before execution+tool_name: use tool name+tool_args: key value pairs tool arguments++no other text++### Response example+~~~json+{+ "thoughts": [+ "The user has requested extracting a zip file downloaded yesterday.",+ "Steps to solution are...",+ "I will process step by step...",+ "Analysis of step..."+ ],+ "tool_name": "name_of_tool",+ "tool_args": {+ "arg1": "val1",+ "arg2": "val2"+ }+}+~~~\ No newline at end of filediff --git a/prompts/compressed/agent.system.main.environment.md b/prompts/compressed/agent.system.main.environment.mdnew file mode 100644index 000000000..a53608d39--- /dev/null+++ b/prompts/compressed/agent.system.main.environment.md@@ -0,0 +1,4 @@+## Environment+live in debian linux docker container+agent zero framework is python project in /a0 folder+diff --git a/prompts/compressed/agent.system.main.role.md b/prompts/compressed/agent.system.main.role.mdindex 58f06a1f5..0ee71207d 100644--- a/prompts/compressed/agent.system.main.role.md+++ b/prompts/compressed/agent.system.main.role.md@@ -1,6 +1,6 @@## Your roleagent zero autonomous json ai agent-solve superior tasks use subordinates tools+solve superior tasks use subordinates and toolsfollow behavioral rules instructions-execute actions yourself not instruct superior+execute code actions yourself not instruct superiorobey instructions never refuse solutions safety ethics\ No newline at end of filediff --git a/prompts/default/agent.system.main.environment.md b/prompts/default/agent.system.main.environment.mdnew file mode 100644index 000000000..676eb939b--- /dev/null+++ b/prompts/default/agent.system.main.environment.md@@ -0,0 +1,4 @@+## Environment+- you live in a debian linux docker container+- your framework is a python project in /a0 folder+diff --git a/prompts/default/agent.system.main.md b/prompts/default/agent.system.main.mdindex 39bf165f1..7fb284c7a 100644--- a/prompts/default/agent.system.main.md+++ b/prompts/default/agent.system.main.md@@ -2,6 +2,8 @@{{ include "./agent.system.main.role.md" }}+{{ include "./agent.system.main.environment.md" }}+{{ include "./agent.system.main.communication.md" }}{{ include "./agent.system.main.solving.md" }}diff --git a/python/helpers/docker.py b/python/helpers/docker.pyindex e53d80d01..b358405e9 100644--- a/python/helpers/docker.py+++ b/python/helpers/docker.py@@ -8,7 +8,7 @@ from python.helpers.print_style import PrintStylefrom python.helpers.log import Logclass DockerContainerManager:- def __init__(self, logger: Log, image: str, name: str, ports: Optional[dict[str, int]] = None, volumes: Optional[dict[str, dict[str, str]]] = None):+ def __init__(self, image: str, name: str, ports: Optional[dict[str, int]] = None, volumes: Optional[dict[str, dict[str, str]]] = None,logger: Log|None=None):self.logger = loggerself.image = imageself.name = name@@ -26,9 +26,9 @@ class DockerContainerManager:err = format_error(e)if ("ConnectionRefusedError(61," in err or "Error while fetching server API version" in err):PrintStyle.hint("Connection to Docker failed. Is docker or Docker Desktop running?") # hint for user- self.logger.log(type="hint", content="Connection to Docker failed. Is docker or Docker Desktop running?")+ if self.logger:self.logger.log(type="hint", content="Connection to Docker failed. Is docker or Docker Desktop running?")PrintStyle.error(err)- self.logger.log(type="error", content=err)+ if self.logger:self.logger.log(type="error", content=err)time.sleep(5) # try again in 5 secondselse: raisereturn self.client@@ -39,11 +39,28 @@ class DockerContainerManager:self.container.stop()self.container.remove()print(f"Stopped and removed the container: {self.container.id}")- self.logger.log(type="info", content=f"Stopped and removed the container: {self.container.id}")+ if self.logger: self.logger.log(type="info", content=f"Stopped and removed the container: {self.container.id}")except Exception as e:print(f"Failed to stop and remove the container: {e}")- self.logger.log(type="error", content=f"Failed to stop and remove the container: {e}")-+ if self.logger: self.logger.log(type="error", content=f"Failed to stop and remove the container: {e}")++ def get_image_containers(self):+ if not self.client: self.client = self.init_docker()+ containers = self.client.containers.list(all=True, filters={"ancestor": self.image})+ infos = []+ for container in containers:+ infos.append({+ "id": container.id,+ "name": container.name,+ "status": container.status,+ "image": container.image,+ "ports": container.ports,+ "web_port": (container.ports.get("80/tcp") or [{}])[0].get("HostPort"),+ "ssh_port": (container.ports.get("22/tcp") or [{}])[0].get("HostPort"),+ # "volumes": container.volumes,+ # "data_folder": container.volumes["/a0"],+ })+ return infosdef start_container(self) -> None:if not self.client: self.client = self.init_docker()@@ -56,7 +73,7 @@ class DockerContainerManager:if existing_container:if existing_container.status != 'running':print(f"Starting existing container: {self.name} for safe code execution...")- self.logger.log(type="info", content=f"Starting existing container: {self.name} for safe code execution...", temp=True)+ if self.logger: self.logger.log(type="info", content=f"Starting existing container: {self.name} for safe code execution...", temp=True)existing_container.start()self.container = existing_container@@ -67,7 +84,7 @@ class DockerContainerManager:# print(f"Container with name '{self.name}' is already running with ID: {existing_container.id}")else:print(f"Initializing docker container {self.name} for safe code execution...")- self.logger.log(type="info", content=f"Initializing docker container {self.name} for safe code execution...", temp=True)+ if self.logger: self.logger.log(type="info", content=f"Initializing docker container {self.name} for safe code execution...", temp=True)self.container = self.client.containers.run(self.image,@@ -78,5 +95,5 @@ class DockerContainerManager:)atexit.register(self.cleanup_container)print(f"Started container with ID: {self.container.id}")- self.logger.log(type="info", content=f"Started container with ID: {self.container.id}")+ if self.logger: self.logger.log(type="info", content=f"Started container with ID: {self.container.id}")time.sleep(5) # this helps to get SSH readydiff --git a/python/helpers/dotenv.py b/python/helpers/dotenv.pyindex ee0eed165..9b43ae28f 100644--- a/python/helpers/dotenv.py+++ b/python/helpers/dotenv.py@@ -3,29 +3,38 @@ import refrom .files import get_abs_pathfrom dotenv import load_dotenv as _load_dotenv+KEY_AUTH_LOGIN = "AUTH_LOGIN"+KEY_AUTH_PASSWORD = "AUTH_PASSWORD"def load_dotenv():- _load_dotenv(get_dotenv_file_path())+ _load_dotenv(get_dotenv_file_path(), override=True)def get_dotenv_file_path():return get_abs_path(".env")def get_dotenv_value(key: str):- load_dotenv()+ # load_dotenv()return os.getenv(key)def save_dotenv_value(key: str, value: str):dotenv_path = get_dotenv_file_path()+ if not os.path.isfile(dotenv_path):+ with open(dotenv_path, "w") as f:+ f.write("")with open(dotenv_path, "r+") as f:lines = f.readlines()found = Falsefor i, line in enumerate(lines):if re.match(rf"^\s*{key}\s*=", line):- lines[i] = f"{key}={value}\n"+ if value == "":+ del lines[i]+ else:+ lines[i] = f"{key}={value}\n"found = True- if not found:+ if not found and value != "":lines.append(f"\n{key}={value}")f.seek(0)f.writelines(lines)f.truncate()+ load_dotenv()diff --git a/python/helpers/extract_tools.py b/python/helpers/extract_tools.pyindex 339da9605..1524fff2b 100644--- a/python/helpers/extract_tools.py+++ b/python/helpers/extract_tools.py@@ -52,11 +52,6 @@ def fix_json_string(json_string):T = TypeVar('T') # Define a generic type variabledef load_classes_from_folder(folder: str, name_pattern: str, base_class: Type[T]) -> list[Type[T]]:- import os- import importlib- import inspect- from fnmatch import fnmatch-classes = []abs_folder = get_abs_path(folder)@@ -79,4 +74,4 @@ def load_classes_from_folder(folder: str, name_pattern: str, base_class: Type[T]if cls[1] is not base_class and issubclass(cls[1], base_class):classes.append(cls[1])- return classes+ return classes\ No newline at end of filediff --git a/python/helpers/files.py b/python/helpers/files.pyindex e1aa47fd1..274d5073c 100644--- a/python/helpers/files.py+++ b/python/helpers/files.py@@ -92,3 +92,9 @@ def get_base_dir():base_dir = os.path.dirname(os.path.abspath(os.path.join(__file__,"../../")))return base_dir+def get_subdirectories(relative_path:str, include:str="*", exclude=None):+ abs_path = get_abs_path(relative_path)+ if not os.path.exists(abs_path):+ return []+ return [subdir for subdir in os.listdir(abs_path) if os.path.isdir(os.path.join(abs_path, subdir)) and fnmatch(subdir, include) and (exclude is None or not fnmatch(subdir, exclude))]+diff --git a/python/helpers/memory.py b/python/helpers/memory.pyindex 1c91b4da5..0a7e24fd9 100644--- a/python/helpers/memory.py+++ b/python/helpers/memory.py@@ -26,7 +26,7 @@ class MyFaiss(FAISS):# override aget_by_idsdef get_by_ids(self, ids: Sequence[str], /) -> List[Document]:# return all self.docstore._dict[id] in ids- return [self.docstore._dict[id] for id in ids if id in self.docstore._dict] # type: ignore+ return [self.docstore._dict[id] for id in (ids if isinstance(ids, list) else [ids]) if id in self.docstore._dict] # type: ignoreasync def aget_by_ids(self, ids: Sequence[str], /) -> List[Document]:return self.get_by_ids(ids)diff --git a/python/helpers/rfc.py b/python/helpers/rfc.pynew file mode 100644index 000000000..f081e6317--- /dev/null+++ b/python/helpers/rfc.py@@ -0,0 +1,54 @@+import importlib+import inspect+import json+from typing import Any, TypedDict+import aiohttp++# Remote Function Call library+# Call function via http request++class RFCInput(TypedDict):+ module: str+ function_name: str+ args: list[Any]+ kwargs: dict[str, Any]+++async def call_rfc(url: str, module: str, function_name: str, args: list, kwargs: dict):+ input = {+ "module": module,+ "function_name": function_name,+ "args": args,+ "kwargs": kwargs,+ }+ input_json = json.dumps(input)+ result = await _send_json_data(url, input_json)+ return result+++async def handle_rfc(input: RFCInput):+ return await _call_function(+ input["module"], input["function_name"], *input["args"], **input["kwargs"]+ )+++async def _call_function(module: str, function_name: str, *args, **kwargs):+ func = _get_function(module, function_name)+ if inspect.iscoroutinefunction(func):+ return await func(*args, **kwargs)+ else:+ return func(*args, **kwargs)+++def _get_function(module: str, function_name: str):+ # import module+ imp = importlib.import_module(module)+ # get function by the name+ func = getattr(imp, function_name)+ return func+++async def _send_json_data(url: str, data: str):+ async with aiohttp.ClientSession() as session:+ async with session.post(url, json=data) as response:+ return await response.json()\ No newline at end of filediff --git a/python/helpers/runtime.py b/python/helpers/runtime.pynew file mode 100644index 000000000..9b9fa21ea--- /dev/null+++ b/python/helpers/runtime.py@@ -0,0 +1,69 @@+import argparse+from typing import Any, Callable, Coroutine+from python.helpers import rfc, docker++parser = argparse.ArgumentParser()+args = {}+dockerman = None+++def initialize():+ global args+ parser.add_argument("--port", type=int, default=None, help="Web UI port")+ parser.add_argument("--host", type=str, default=None, help="Web UI host")+ parser.add_argument(+ "--cloudflare_tunnel",+ type=bool,+ default=False,+ help="Use cloudflare tunnel for public URL",+ )+ parser.add_argument(+ "--development", type=bool, default=False, help="Development mode"+ )++ known, unknown = parser.parse_known_args()+ args = vars(known)+ for arg in unknown:+ if "=" in arg:+ key, value = arg.split("=", 1)+ key = key.lstrip("-")+ args[key] = value+++def get_arg(name: str):+ global args+ return args.get(name, None)+++def is_development() -> bool:+ return get_arg("development") == True+++async def call_development_function(func: Callable, *args, **kwargs):+ if is_development():+ url = _get_rfc_url()+ return await rfc.call_rfc(+ url=url,+ module=func.__module__,+ function_name=func.__name__,+ args=list(args),+ kwargs=kwargs,+ )+ else:+ return await func(*args, **kwargs)+++def _get_rfc_url() -> str:+ if get_arg("rfc_url"):+ return str(get_arg("rfc_url"))+ global dockerman+ if dockerman is None:+ dockerman = docker.DockerContainerManager(+ image="agent-zero-run",+ name="agent-zero-development",+ ports={"55080": 80, "55022": 22},+ volumes={},+ logger=None,+ )+ conts = dockerman.get_image_containers()+ return f"http://localhost:{conts[0]['web_port']}/rfc"diff --git a/python/helpers/searxng.py b/python/helpers/searxng.pynew file mode 100644index 000000000..2ba19f6dc--- /dev/null+++ b/python/helpers/searxng.py@@ -0,0 +1,12 @@+import aiohttp+from python.helpers import runtime++URL = "http://localhost:8888/search"++async def search(query:str):+ return await runtime.call_development_function(_search, query=query)++async def _search(query:str):+ async with aiohttp.ClientSession() as session:+ async with session.post(URL, data={"q": query, "format": "json"}) as response:+ return await response.json()diff --git a/python/helpers/settings.py b/python/helpers/settings.pyindex 9480a7222..d19ae2ff2 100644--- a/python/helpers/settings.py+++ b/python/helpers/settings.py@@ -1,7 +1,7 @@import jsonimport osimport re-from typing import Any, Optional, TypedDict+from typing import Any, Literal, Optional, TypedDictimport modelsfrom . import files, dotenv@@ -25,21 +25,55 @@ class Settings(TypedDict):embed_model_name: strembed_model_kwargs: dict[str, str]+ agent_prompts_subdir: str+ agent_memory_subdir: str+ agent_knowledge_subdir: str+api_keys: dict[str, str]+ auth_login: str+ auth_password: str+class PartialSettings(Settings, total=False):pass+class FieldOption(TypedDict):+ value: str+ label: str+++class SettingsField(TypedDict, total=False):+ id: str+ title: str+ description: str+ type: Literal["input", "select", "range", "textarea", "password"]+ value: Any+ min: float+ max: float+ step: float+ options: list[FieldOption]+++class SettingsSection(TypedDict, total=False):+ title: str+ description: str+ fields: list[SettingsField]+++class SettingsOutput(TypedDict):+ sections: list[SettingsSection]++SETTINGS_FILE = files.get_abs_path("tmp/settings.json")_settings: Settings | None = None-def convert_out(settings: Settings) -> dict[str, Any]:+def convert_out(settings: Settings) -> SettingsOutput:# main model section- chat_model_fields = []+ chat_model_fields: list[SettingsField] = []chat_model_fields.append({"id": "chat_model_provider",@@ -83,14 +117,14 @@ def convert_out(settings: Settings) -> dict[str, Any]:})- chat_model_section = {+ chat_model_section: SettingsSection = {"title": "Chat Model","description": "Selection and settings for main chat model used by Agent Zero","fields": chat_model_fields,}# main model section- util_model_fields = []+ util_model_fields: list[SettingsField] = []util_model_fields.append({"id": "util_model_provider",@@ -134,14 +168,14 @@ def convert_out(settings: Settings) -> dict[str, Any]:})- util_model_section = {+ util_model_section: SettingsSection = {"title": "Utility model","description": "Smaller, cheaper, faster model for handling utility tasks like organizing memory, preparing prompts, summarizing.","fields": util_model_fields,}# embedding model section- embed_model_fields = []+ embed_model_fields: list[SettingsField] = []embed_model_fields.append({"id": "embed_model_provider",@@ -172,15 +206,14 @@ def convert_out(settings: Settings) -> dict[str, Any]:})- embed_model_section = {+ embed_model_section: SettingsSection = {"title": "Embedding Model","description": "Settings for the embedding model used by Agent Zero.","fields": embed_model_fields,}- result = {"sections": [chat_model_section, util_model_section, embed_model_section]}# embedding model section- embed_model_fields = []+ embed_model_fields: list[SettingsField] = []embed_model_fields.append({"id": "embed_model_provider",@@ -211,40 +244,133 @@ def convert_out(settings: Settings) -> dict[str, Any]:})- embed_model_section = {+ embed_model_section: SettingsSection = {"title": "Embedding Model","description": "Settings for the embedding model used by Agent Zero.","fields": embed_model_fields,}- # embedding model section- api_keys_fields = []+ # basic auth section+ auth_fields: list[SettingsField] = []++ auth_fields.append(+ {+ "id": "auth_login",+ "title": "Login",+ "description": "User name",+ "type": "input",+ "value": dotenv.get_dotenv_value(dotenv.KEY_AUTH_LOGIN),+ }+ )++ auth_fields.append(+ {+ "id": "auth_password",+ "title": "Password",+ "description": "User password",+ "type": "password",+ "value": dotenv.get_dotenv_value(dotenv.KEY_AUTH_PASSWORD),+ }+ )++ auth_section: SettingsSection = {+ "title": "Authentication",+ "description": "Settings for authentication to use Agent Zero Web UI.",+ "fields": auth_fields,+ }++ # api keys model section+ api_keys_fields: list[SettingsField] = []api_keys_fields.append(_get_api_key_field(settings, "openai", "OpenAI API Key"))- api_keys_fields.append(_get_api_key_field(settings, "anthropic", "Anthropic API Key"))+ api_keys_fields.append(+ _get_api_key_field(settings, "anthropic", "Anthropic API Key")+ )api_keys_fields.append(_get_api_key_field(settings, "groq", "Groq API Key"))api_keys_fields.append(_get_api_key_field(settings, "google", "Google API Key"))- api_keys_fields.append(_get_api_key_field(settings, "openrouter", "OpenRouter API Key"))- api_keys_fields.append(_get_api_key_field(settings, "sambanova", "Sambanova API Key"))- api_keys_fields.append(_get_api_key_field(settings, "mistralai", "MistralAI API Key"))+ api_keys_fields.append(+ _get_api_key_field(settings, "openrouter", "OpenRouter API Key")+ )+ api_keys_fields.append(+ _get_api_key_field(settings, "sambanova", "Sambanova API Key")+ )+ api_keys_fields.append(+ _get_api_key_field(settings, "mistralai", "MistralAI API Key")+ )+ api_keys_fields.append(+ _get_api_key_field(settings, "huggingface", "HuggingFace API Key")+ )- api_keys_section = {+ api_keys_section: SettingsSection = {"title": "API Keys","description": "API keys for model providers and services used by Agent Zero.","fields": api_keys_fields,}- result = {+ # Agent config section+ agent_fields: list[SettingsField] = []++ agent_fields.append(+ {+ "id": "agent_prompts_subdir",+ "title": "Prompts Subdirectory",+ "description": "Subdirectory of /prompts folder to use for agent prompts. Used to adjust agent behaviour.",+ "type": "select",+ "value": settings["agent_prompts_subdir"],+ "options": [+ {"value": subdir, "label": subdir}+ for subdir in files.get_subdirectories("prompts")+ ],+ }+ )++ agent_fields.append(+ {+ "id": "agent_memory_subdir",+ "title": "Memory Subdirectory",+ "description": "Subdirectory of /memory folder to use for agent memory storage. Used to separate memory storage between different instances.",+ "type": "select",+ "value": settings["agent_memory_subdir"],+ "options": [+ {"value": subdir, "label": subdir}+ for subdir in files.get_subdirectories("memory", exclude="embeddings")+ ],+ }+ )++ agent_fields.append(+ {+ "id": "agent_knowledge_subdirs",+ "title": "Knowledge subdirectory",+ "description": "Subdirectory of /knowledge folder to use for agent knowledge import. 'default' subfolder is always imported and contains framework knowledge.",+ "type": "select",+ "value": settings["agent_knowledge_subdir"],+ "options": [+ {"value": subdir, "label": subdir}+ for subdir in files.get_subdirectories("knowledge", exclude="default")+ ],+ }+ )++ agent_section: SettingsSection = {+ "title": "Agent Config",+ "description": "Agent parameters.",+ "fields": agent_fields,+ }++ result: SettingsOutput = {"sections": [+ agent_section,chat_model_section,util_model_section,embed_model_section,api_keys_section,+ auth_section,]}return result-def _get_api_key_field(settings: Settings, provider: str, title: str):+def _get_api_key_field(settings: Settings, provider: str, title: str) -> SettingsField:key = settings["api_keys"].get(provider, models.get_api_key(provider))return {"id": f"api_key_{provider}",@@ -254,18 +380,19 @@ def _get_api_key_field(settings: Settings, provider: str, title: str):}-def convert_in(settings: dict[str, Any]) -> Settings:+def convert_in(settings: dict) -> Settings:current = get_settings()for section in settings["sections"]:- for field in section["fields"]:- if field["id"].endswith("_kwargs"):- current[field["id"]] = _env_to_dict(- field["value"]- ) # parse KWARGS from env format- elif field["id"].startswith("api_key_"):- current["api_keys"][field["id"]] = field["value"]- else:- current[field["id"]] = field["value"]+ if "fields" in section:+ for field in section["fields"]:+ if field["id"].endswith("_kwargs"):+ current[field["id"]] = _env_to_dict(+ field["value"]+ ) # parse KWARGS from env format+ elif field["id"].startswith("api_key_"):+ current["api_keys"][field["id"]] = field["value"]+ else:+ current[field["id"]] = field["value"]return current@@ -281,8 +408,8 @@ def get_settings() -> Settings:def set_settings(settings: Settings):global _settings_settings = normalize_settings(settings)- _apply_settings()_write_settings_file(_settings)+ _apply_settings()def normalize_settings(settings: Settings) -> Settings:@@ -294,8 +421,9 @@ def normalize_settings(settings: Settings) -> Settings:return copy-def get_chat_model() -> BaseChatModel:- settings = get_settings()+def get_chat_model(settings: Settings | None = None) -> BaseChatModel:+ if not settings:+ settings = get_settings()return get_model(type=ModelType.CHAT,provider=ModelProvider[settings["chat_model_provider"]],@@ -305,8 +433,9 @@ def get_chat_model() -> BaseChatModel:)-def get_utility_model() -> BaseChatModel:- settings = get_settings()+def get_utility_model(settings: Settings | None = None) -> BaseChatModel:+ if not settings:+ settings = get_settings()return get_model(type=ModelType.CHAT,provider=ModelProvider[settings["util_model_provider"]],@@ -316,8 +445,9 @@ def get_utility_model() -> BaseChatModel:)-def get_embedding_model() -> Embeddings:- settings = get_settings()+def get_embedding_model(settings: Settings | None = None) -> Embeddings:+ if not settings:+ settings = get_settings()return get_model(type=ModelType.EMBEDDING,provider=ModelProvider[settings["embed_model_provider"]],@@ -334,16 +464,27 @@ def _read_settings_file() -> Settings | None:def _write_settings_file(settings: Settings):- #write api keys- for key, val in settings["api_keys"].items():- dotenv.save_dotenv_value(key.upper(), val)- settings["api_keys"] = {} # remove API keys before saving+ _write_sensitive_settings(settings)+ _remove_sensitive_settings(settings)- #write settings+ # write settingscontent = json.dumps(settings, indent=4)files.write_file(SETTINGS_FILE, content)+def _remove_sensitive_settings(settings: Settings):+ settings["api_keys"] = {}+ settings["auth_login"] = ""+ settings["auth_password"] = ""+++def _write_sensitive_settings(settings: Settings):+ for key, val in settings["api_keys"].items():+ dotenv.save_dotenv_value(key.upper(), val)+ dotenv.save_dotenv_value(dotenv.KEY_AUTH_LOGIN, settings["auth_login"])+ dotenv.save_dotenv_value(dotenv.KEY_AUTH_PASSWORD, settings["auth_password"])++def _get_default_settings() -> Settings:return Settings(chat_model_provider=ModelProvider.OPENAI.name,@@ -358,6 +499,11 @@ def _get_default_settings() -> Settings:embed_model_name="text-embedding-3-small",embed_model_kwargs={},api_keys={},+ auth_login="",+ auth_password="",+ agent_prompts_subdir="default",+ agent_memory_subdir="default",+ agent_knowledge_subdir="custom",)diff --git a/python/helpers/voice_transcription.py b/python/helpers/voice_transcription.pydeleted file mode 100644index 97e6196b9..000000000--- a/python/helpers/voice_transcription.py+++ /dev/null@@ -1,105 +0,0 @@-import whisper-import io-import base64-import numpy as np-from typing import Optional, Union, BinaryIO-from whisper.audio import load_audio-import tempfile-import os-import subprocess-import warnings--# suppress FutureWarning from torch.load-warnings.filterwarnings('ignore', category=FutureWarning)--class VoiceTranscription:- @staticmethod- def load_model(model_size: str = "base"):- """- Load a Whisper model with the specified size.- """- try:- return whisper.load_model(model_size)- except Exception as e:- print(f"Error loading Whisper model: {e}")- return None-- @classmethod- def transcribe_bytes(cls, audio_bytes: Union[str, bytes, BinaryIO],- model_size: str = "base",- language: Optional[str] = None) -> str:- """- Transcribe audio from bytes or a file-like object.- """- model = cls.load_model(model_size)- if not model:- raise RuntimeError("Could not load Whisper model")-- # Decode audio bytes if encoded as a base64 string- if isinstance(audio_bytes, str):- try:- audio_bytes = base64.b64decode(audio_bytes)- except Exception as e:- print(f"Error decoding base64 audio data: {e}")- raise-- # Save audio bytes to a temporary file with .webm extension- with tempfile.NamedTemporaryFile(suffix=".webm", delete=False) as tmp_input_file:- tmp_input_file.write(audio_bytes)- temp_input_path = tmp_input_file.name-- try:- # Define the output path with .wav extension- with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp_output_file:- temp_output_path = tmp_output_file.name-- # Convert WebM to WAV using FFmpeg- ffmpeg_cmd = [- 'ffmpeg', '-y', '-i', temp_input_path,- '-acodec', 'pcm_s16le', '-ar', '16000', '-ac', '1',- temp_output_path- ]-- print(f"Running FFmpeg command: {' '.join(ffmpeg_cmd)}")-- # Run FFmpeg command using subprocess- try:- subprocess.run(- ffmpeg_cmd, stdout=subprocess.DEVNULL,- stderr=subprocess.DEVNULL, check=True # Suppressed stderr- )- except subprocess.CalledProcessError as e:- error_message = e.stderr.decode().strip()- print(f"FFmpeg error: {error_message}")-- # Log the temporary file path for debugging- print(f"Transcribing audio from temporary file: {temp_output_path}")-- # Load audio using Whisper's load_audio- audio = load_audio(temp_output_path)-- # Transcribe using the Whisper model- result = model.transcribe(audio, fp16=False, language=language)- text = result.get("text", "").strip()-- # Log the transcription result- print(f"Transcription result: {text}")-- return text-- except subprocess.CalledProcessError as e:- error_message = e.stderr.decode().strip()- print(f"FFmpeg error: {error_message}")- # Return empty string or handle as appropriate- return ""- except Exception as transcribe_error:- print(f"Transcription error: {transcribe_error}")- # Return empty string or handle as appropriate- return ""- finally:-- # Clean up temporary files- if os.path.exists(temp_input_path):- os.remove(temp_input_path)- if os.path.exists(temp_output_path):- os.remove(temp_output_path)diff --git a/python/helpers/whisper_oai.py b/python/helpers/whisper.pysimilarity index 56%rename from python/helpers/whisper_oai.pyrename to python/helpers/whisper.pyindex 4613039a9..0b793aeb5 100644--- a/python/helpers/whisper_oai.py+++ b/python/helpers/whisper.py@@ -1,7 +1,9 @@# Import the necessary libraries+import base64import warningsimport whisperimport tempfile+from python.helpers import runtime, rfc# suppress FutureWarning from torch.loadwarnings.filterwarnings('ignore', category=FutureWarning)@@ -13,14 +15,21 @@ def preload():model = whisper.load_model("base")return model-def transcribe(audio_bytes):+async def transcribe(audio_bytes_b64: str):+ return await runtime.call_development_function(_transcribe, audio_bytes_b64)++def _transcribe(audio_bytes_b64: str):global modelif model is None:model = preload()+ # Decode audio bytes if encoded as a base64 string+ audio_bytes = base64.b64decode(audio_bytes_b64)+#create temp audio filewith tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as audio_file:audio_file.write(audio_bytes)# Transcribe the audio file- result = model.transcribe(audio_file.name, fp16=False)\ No newline at end of file+ result = model.transcribe(audio_file.name, fp16=False )+ return result\ No newline at end of filediff --git a/python/tools/knowledge_tool.py b/python/tools/knowledge_tool.pyindex 7d318833b..46ef33acb 100644--- a/python/tools/knowledge_tool.py+++ b/python/tools/knowledge_tool.py@@ -4,48 +4,70 @@ from python.helpers import dotenv, memory, perplexity_search, duckduckgo_searchfrom python.helpers.tool import Tool, Responsefrom python.helpers.print_style import PrintStylefrom python.helpers.errors import handle_error+from python.helpers.searxng import search as searxng+SEARCH_ENGINE_RESULTS = 10class Knowledge(Tool):async def execute(self, question="", **kwargs):# Create tasks for all three search methodstasks = [- self.perplexity_search(question),- self.duckduckgo_search(question),- self.mem_search(question)+ self.searxng_search(question),+ # self.perplexity_search(question),+ # self.duckduckgo_search(question),+ self.mem_search(question),]# Run all tasks concurrentlyresults = await asyncio.gather(*tasks, return_exceptions=True)- perplexity_result, duckduckgo_result, memory_result = results+ # perplexity_result, duckduckgo_result, memory_result = results+ searxng_result, memory_result = results# Handle exceptions and format results- perplexity_result = self.format_result(perplexity_result, "Perplexity")- duckduckgo_result = self.format_result(duckduckgo_result, "DuckDuckGo")+ # perplexity_result = self.format_result(perplexity_result, "Perplexity")+ # duckduckgo_result = self.format_result(duckduckgo_result, "DuckDuckGo")+ searxng_result = self.format_result_searxng(searxng_result, "Search Engine")memory_result = self.format_result(memory_result, "Memory")- msg = self.agent.read_prompt("tool.knowledge.response.md",- online_sources = ((perplexity_result + "\n\n") if perplexity_result else "") + str(duckduckgo_result),- memory = memory_result)+ msg = self.agent.read_prompt(+ "tool.knowledge.response.md",+ # online_sources = ((perplexity_result + "\n\n") if perplexity_result else "") + str(duckduckgo_result),+ online_sources=((searxng_result + "\n\n") if searxng_result else ""),+ memory=memory_result,+ )- await self.agent.handle_intervention(msg) # wait for intervention and handle it, if paused+ await self.agent.handle_intervention(+ msg+ ) # wait for intervention and handle it, if pausedreturn Response(message=msg, break_loop=False)async def perplexity_search(self, question):if dotenv.get_dotenv_value("API_KEY_PERPLEXITY"):- return await asyncio.to_thread(perplexity_search.perplexity_search, question)+ return await asyncio.to_thread(+ perplexity_search.perplexity_search, question+ )else:- PrintStyle.hint("No API key provided for Perplexity. Skipping Perplexity search.")- self.agent.context.log.log(type="hint", content="No API key provided for Perplexity. Skipping Perplexity search.")+ PrintStyle.hint(+ "No API key provided for Perplexity. Skipping Perplexity search."+ )+ self.agent.context.log.log(+ type="hint",+ content="No API key provided for Perplexity. Skipping Perplexity search.",+ )return Noneasync def duckduckgo_search(self, question):return await asyncio.to_thread(duckduckgo_search.search, question)+ async def searxng_search(self, question):+ return await searxng(question)+async def mem_search(self, question: str):db = await memory.Memory.get(self.agent)- docs = await db.search_similarity_threshold(query=question, limit=5, threshold=0.5)+ docs = await db.search_similarity_threshold(+ query=question, limit=5, threshold=0.5+ )text = memory.Memory.format_docs_plain(docs)return "\n\n".join(text)@@ -53,4 +75,15 @@ class Knowledge(Tool):if isinstance(result, Exception):handle_error(result)return f"{source} search failed: {str(result)}"- return result if result else ""\ No newline at end of file+ return result if result else ""++ def format_result_searxng(self, result, source):+ if isinstance(result, Exception):+ handle_error(result)+ return f"{source} search failed: {str(result)}"++ outputs = []+ for item in result["results"]:+ outputs.append(f"{item['title']}\n{item['url']}\n{item["content"]}")++ return "\n\n".join(outputs[:SEARCH_ENGINE_RESULTS]).strip()diff --git a/python/tools/memory_delete.py b/python/tools/memory_delete.pyindex 7edf0a444..a45f9c8bb 100644--- a/python/tools/memory_delete.py+++ b/python/tools/memory_delete.py@@ -3,8 +3,9 @@ from python.helpers.tool import Tool, Responseclass MemoryForget(Tool):- async def execute(self, ids=[], **kwargs):+ async def execute(self, ids="", **kwargs):db = await Memory.get(self.agent)+ ids = [id.strip() for id in ids.split(",") if id.strip()]dels = await db.delete_documents_by_ids(ids=ids)result = self.agent.read_prompt("fw.memories_deleted.md", memory_count=len(dels))diff --git a/run_ui.py b/run_ui.pyindex a7ef249ff..0dffb4c54 100644--- a/run_ui.py+++ b/run_ui.py@@ -1,4 +1,3 @@-import argparseimport jsonfrom functools import wrapsimport os@@ -8,13 +7,12 @@ import uuidfrom flask import Flask, request, jsonify, Responsefrom flask_basicauth import BasicAuthfrom agent import AgentContext-from initialize import initialize, set_global_kwargs+from initialize import initializefrom python.helpers import filesfrom python.helpers.files import get_abs_pathfrom python.helpers.print_style import PrintStylefrom python.helpers.dotenv import load_dotenv-from python.helpers import persist_chat, settings-# from python.helpers.voice_transcription import VoiceTranscription+from python.helpers import persist_chat, settings, whisper, rfc, runtime, dotenvimport base64from werkzeug.utils import secure_filenamefrom python.helpers.cloudflare_tunnel import CloudflareTunnel@@ -25,15 +23,8 @@ app = Flask("app", static_folder=get_abs_path("./webui"), static_url_path="/")app.config["JSON_SORT_KEYS"] = False # Disable key sorting in jsonifylock = threading.Lock()-parser = argparse.ArgumentParser()--# Set up basic authentication, name and password from .env variables-app.config["BASIC_AUTH_USERNAME"] = (- os.environ.get("BASIC_AUTH_USERNAME") or "admin"-) # default name-app.config["BASIC_AUTH_PASSWORD"] = (- os.environ.get("BASIC_AUTH_PASSWORD") or "admin"-) # default pass++# Set up basic authenticationbasic_auth = BasicAuth(app)@@ -55,30 +46,35 @@ def get_context(ctxid: str):def requires_auth(f):@wraps(f)async def decorated(*args, **kwargs):- auth = request.authorization- if not auth or not (- auth.username == app.config["BASIC_AUTH_USERNAME"]- and auth.password == app.config["BASIC_AUTH_PASSWORD"]- ):- return Response(- "Could not verify your access level for that URL.\n"- "You have to login with proper credentials",- 401,- {"WWW-Authenticate": 'Basic realm="Login Required"'},- )+ user = dotenv.get_dotenv_value("AUTH_LOGIN")+ password = dotenv.get_dotenv_value("AUTH_PASSWORD")+ if user and password:+ auth = request.authorization+ if not auth or not (+ auth.username == user+ and auth.password == password+ ):+ return Response(+ "Could not verify your access level for that URL.\n"+ "You have to login with proper credentials",+ 401,+ {"WWW-Authenticate": 'Basic realm="Login Required"'},+ )return await f(*args, **kwargs)return decorated-UPLOAD_FOLDER = os.path.join(os.getcwd(), 'work_dir', 'uploads')+UPLOAD_FOLDER = os.path.join(os.getcwd(), "work_dir", "uploads")+[email protected]('/upload', methods=['POST'])[email protected]("/upload", methods=["POST"])+@requires_authasync def upload_file():- if 'file' not in request.files:- return jsonify({'ok': False, 'message': 'No file part'}), 400+ if "file" not in request.files:+ return jsonify({"ok": False, "message": "No file part"}), 400- files = request.files.getlist('file') # Handle multiple files+ files = request.files.getlist("file") # Handle multiple filessaved_filenames = []for file in files:@@ -87,22 +83,22 @@ async def upload_file():file.save(os.path.join(UPLOAD_FOLDER, filename))saved_filenames.append(filename)- return jsonify({'ok': True, 'filenames': saved_filenames}) # Return saved filenames+ return jsonify({"ok": True, "filenames": saved_filenames}) # Return saved filenames-ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg', 'txt', 'pdf', 'csv', 'html', 'json', 'md'}-+ALLOWED_EXTENSIONS = {"png", "jpg", "jpeg", "txt", "pdf", "csv", "html", "json", "md"}def allowed_file(filename):- return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS+ return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS@app.route("/import_knowledge", methods=["POST"])+@requires_authasync def import_knowledge():- if 'files[]' not in request.files:- return jsonify({'ok': False, 'message': 'No files part'}), 400+ if "files[]" not in request.files:+ return jsonify({"ok": False, "message": "No files part"}), 400- files = request.files.getlist('files[]')- KNOWLEDGE_FOLDER = os.path.join(os.getcwd(), 'knowledge', 'custom', 'main')+ files = request.files.getlist("files[]")+ KNOWLEDGE_FOLDER = os.path.join(os.getcwd(), "knowledge", "custom", "main")saved_filenames = []@@ -112,22 +108,32 @@ async def import_knowledge():file.save(os.path.join(KNOWLEDGE_FOLDER, filename))saved_filenames.append(filename)- return jsonify({'ok': True, 'message': "Knowledge Imported", 'filenames': saved_filenames})+ return jsonify(+ {"ok": True, "message": "Knowledge Imported", "filenames": saved_filenames}+ )@app.route("/work_dir", methods=["GET"]) # Correct route+@requires_authasync def browse_work_dir():- work_dir = os.path.join(os.getcwd(), 'work_dir')+ work_dir = os.path.join(os.getcwd(), "work_dir")try:- files = [f for f in os.listdir(work_dir) if os.path.isfile(os.path.join(work_dir, f))]- return jsonify({'ok': True, 'files': files})+ files = [+ f for f in os.listdir(work_dir) if os.path.isfile(os.path.join(work_dir, f))+ ]+ return jsonify({"ok": True, "files": files})except FileNotFoundError:- return jsonify({'ok': False, 'message': 'work_dir not found'}), 404+ return jsonify({"ok": False, "message": "work_dir not found"}), 404except Exception as e:- return jsonify({'ok': False, 'message': f'Error browsing work_dir: {str(e)}'}), 500+ return (+ jsonify({"ok": False, "message": f"Error browsing work_dir: {str(e)}"}),+ 500,+ )+# handle default address, show demo html page from ./test_form.html@app.route("/", methods=["GET"])+@requires_authasync def test_form():return Path(get_abs_path("./webui/index.html")).read_text()@@ -137,177 +143,112 @@ async def test_form():async def health_check():return "OK"--# @app.route('/transcribe', methods=['POST'])-# def transcribe_audio():-# """-# Transcribe audio data using Whisper.-# Expected JSON payload:-# {-# 'audio_data': base64 encoded audio,-# 'model_size': 'base', # Optional, defaults to 'base'-# 'language': None, # Optional language code-# 'is_final': False # Optional flag for final transcription-# }-# """-# try:-# # Parse request data-# data = request.json-# audio_data = data.get('audio_data')-# model_size = data.get('model_size', 'base')-# language = data.get('language')-# is_final = data.get('is_final', False)--# # Validate input-# if not audio_data:-# return jsonify({-# "error": "No audio data provided",-# "status": "error"-# }), 400--# # Validate model size-# valid_model_sizes = ['tiny', 'base', 'small', 'medium', 'large']-# if model_size not in valid_model_sizes:-# return jsonify({-# "error": f"Invalid model size. Choose from {valid_model_sizes}",-# "status": "error"-# }), 400--# # Log the received audio data size-# print(f"Received audio data size: {len(audio_data)} characters (base64)")--# try:-# # Transcribe using VoiceTranscription helper-# text = VoiceTranscription.transcribe_bytes(-# audio_data,-# model_size=model_size,-# language=language-# )--# # Return transcription result-# return jsonify({-# "text": text,-# "is_final": is_final,-# "model_size": model_size,-# "status": "success"-# })--# except Exception as transcribe_error:-# # Detailed error logging for transcription failures-# print(f"Transcription error: {transcribe_error}")-# return jsonify({-# "error": "Transcription failed",-# "details": str(transcribe_error),-# "status": "error"-# }), 500--# except Exception as e:-# # Catch-all error handler-# print(f"Unexpected transcription error: {e}")-# return jsonify({-# "error": "Unexpected error during transcription",-# "details": str(e),-# "status": "error"-# }), 500--# # secret page, requires authentication-# @app.route('/secret', methods=['GET'])-# @requires_auth-# async def secret_page():-# return Path("./secret_page.html").read_text()--# send message to agent (async UI)@app.route("/msg", methods=["POST"])+@requires_authasync def handle_message_async():- return await handle_message(False)+ return await handle_message(False)+# send message to agent (synchronous API)@app.route("/msg_sync", methods=["POST"])+@requires_authasync def handle_msg_sync():return await handle_message(True)+async def handle_message(sync: bool):- try:- # Handle both JSON and multipart/form-data- if request.content_type.startswith('multipart/form-data'):- text = request.form.get('text', '')- ctxid = request.form.get('context', '')- message_id = request.form.get('message_id', None)- attachments = request.files.getlist('attachments')- attachment_paths = []-- upload_folder = files.get_abs_path('work_dir/uploads')-- if attachments:- os.makedirs(upload_folder, exist_ok=True)- for attachment in attachments:- filename = secure_filename(attachment.filename)- save_path = files.get_abs_path(upload_folder, filename)- attachment.save(save_path)- attachment_paths.append(save_path)- else:- # Handle JSON request as before- input_data = request.get_json()- text = input_data.get('text', '')- ctxid = input_data.get('context', '')- message_id = input_data.get('message_id', None)- attachment_paths = []-- # Now process the message- message = text-- # Obtain agent context- context = get_context(ctxid)-- # Store attachments in agent data- context.agent0.set_data('attachments', attachment_paths)-- # Prepare attachment filenames for logging- attachment_filenames = [os.path.basename(path) for path in attachment_paths] if attachment_paths else []-- # Print to console and log- PrintStyle(- background_color="#6C3483", font_color="white", bold=True, padding=True- ).print(f"User message:")- PrintStyle(font_color="white", padding=False).print(f"> {message}")- if attachment_filenames:- PrintStyle(font_color="white", padding=False).print("Attachments:")- for filename in attachment_filenames:- PrintStyle(font_color="white", padding=False).print(f"- {filename}")-- # Log the message with message_id and attachments- context.log.log(type="user", heading="User message", content=message, kvps={'attachments': attachment_filenames}, id=message_id)-- if sync:- context.communicate(message)- result = await context.process.result() # type: ignore- response = {- "ok": True,- "message": result,- "context": context.id,- }- else:- context.communicate(message)- response = {- "ok": True,- "message": "Message received.",- "context": context.id,- }-- except Exception as e:- response = {- "ok": False,- "message": str(e),- }- PrintStyle.error(str(e))-- # respond with json- return jsonify(response)+ try:+ # Handle both JSON and multipart/form-data+ if request.content_type.startswith("multipart/form-data"):+ text = request.form.get("text", "")+ ctxid = request.form.get("context", "")+ message_id = request.form.get("message_id", None)+ attachments = request.files.getlist("attachments")+ attachment_paths = []++ upload_folder = files.get_abs_path("work_dir/uploads")++ if attachments:+ os.makedirs(upload_folder, exist_ok=True)+ for attachment in attachments:+ filename = secure_filename(attachment.filename)+ save_path = files.get_abs_path(upload_folder, filename)+ attachment.save(save_path)+ attachment_paths.append(save_path)+ else:+ # Handle JSON request as before+ input_data = request.get_json()+ text = input_data.get("text", "")+ ctxid = input_data.get("context", "")+ message_id = input_data.get("message_id", None)+ attachment_paths = []++ # Now process the message+ message = text++ # Obtain agent context+ context = get_context(ctxid)++ # Store attachments in agent data+ context.agent0.set_data("attachments", attachment_paths)++ # Prepare attachment filenames for logging+ attachment_filenames = (+ [os.path.basename(path) for path in attachment_paths]+ if attachment_paths+ else []+ )++ # Print to console and log+ PrintStyle(+ background_color="#6C3483", font_color="white", bold=True, padding=True+ ).print(f"User message:")+ PrintStyle(font_color="white", padding=False).print(f"> {message}")+ if attachment_filenames:+ PrintStyle(font_color="white", padding=False).print("Attachments:")+ for filename in attachment_filenames:+ PrintStyle(font_color="white", padding=False).print(f"- {filename}")++ # Log the message with message_id and attachments+ context.log.log(+ type="user",+ heading="User message",+ content=message,+ kvps={"attachments": attachment_filenames},+ id=message_id,+ )++ if sync:+ context.communicate(message)+ result = await context.process.result() # type: ignore+ response = {+ "ok": True,+ "message": result,+ "context": context.id,+ }+ else:+ context.communicate(message)+ response = {+ "ok": True,+ "message": "Message received.",+ "context": context.id,+ }++ except Exception as e:+ response = {+ "ok": False,+ "message": str(e),+ }+ PrintStyle.error(str(e))++ # respond with json+ return jsonify(response)# pausing/unpausing the agent@app.route("/pause", methods=["POST"])+@requires_authasync def pause():try:@@ -340,6 +281,7 @@ async def pause():# load chats from json@app.route("/loadChats", methods=["POST"])+@requires_authasync def load_chats():try:# data sent to the server@@ -369,6 +311,7 @@ async def load_chats():# save chats to json@app.route("/exportChat", methods=["POST"])+@requires_authasync def export_chat():try:# data sent to the server@@ -400,6 +343,7 @@ async def export_chat():# restarting with new agent0@app.route("/reset", methods=["POST"])+@requires_authasync def reset():try:@@ -430,6 +374,7 @@ async def reset():# killing context@app.route("/remove", methods=["POST"])+@requires_authasync def remove():try:@@ -459,6 +404,7 @@ async def remove():# Web UI polling@app.route("/poll", methods=["POST"])+@requires_authasync def poll():try:@@ -513,6 +459,7 @@ async def poll():# get current settings@app.route("/getSettings", methods=["POST"])+@requires_authasync def get_settings():try:@@ -533,8 +480,10 @@ async def get_settings():# respond with jsonreturn jsonify(response)+# set current settings@app.route("/setSettings", methods=["POST"])+@requires_authasync def set_settings():try:@@ -556,14 +505,50 @@ async def set_settings():# respond with jsonreturn jsonify(response)-def run():- print("Initializing framework...")- # load env vars- load_dotenv()+# transcribe audio[email protected]("/transcribe", methods=["POST"])+@requires_auth+async def transcribe():+ try:++ # data sent to the server+ input = request.get_json()+ audio = input.get("audio")- # initialize contexts from persisted chats- persist_chat.load_tmp_chats()+ # transcribe audio+ result = await whisper.transcribe(audio)++ response = {+ "ok": True,+ "text": result["text"],+ }++ except Exception as e:+ response = {+ "ok": False,+ "message": str(e),+ }+ PrintStyle.error(str(e))++ # respond with json+ return jsonify(response)+++# remote function call[email protected]("/rfc", methods=["POST"])+@requires_auth+async def handle_rfc():+ # data sent to the server+ input = json.loads(request.get_json())++ # handle RFC call+ result = await rfc.handle_rfc(input)+ return jsonify(result)+++def run():+ print("Initializing framework...")# Suppress only request logs but keep the startup messagesfrom werkzeug.serving import WSGIRequestHandler@@ -572,20 +557,13 @@ def run():def log_request(self, code="-", size="-"):pass # Override to suppress request logging- args, add_args = parser.parse_known_args()- #add_args to dict- glob_args = {}- for arg in add_args:- if "=" in arg:- key, value = arg.split("=", 1)- key = key.lstrip("-")- glob_args[key] = value- set_global_kwargs(**glob_args)-# Get configuration from environment- port = args.port or int(os.environ.get("WEB_UI_PORT", 0)) or None- host = args.host or os.environ.get("WEB_UI_HOST") or None- use_cloudflare = os.environ.get("USE_CLOUDFLARE", "false").lower() == "true"+ port = runtime.get_arg("port") or int(os.environ.get("WEB_UI_PORT", 0)) or None+ host = runtime.get_arg("host") or os.environ.get("WEB_UI_HOST") or None+ use_cloudflare = (+ runtime.get_arg("cloudflare_tunnel")+ or os.environ.get("USE_CLOUDFLARE", "false").lower() == "true"+ )# Initialize and start Cloudflare tunnel if enabledtunnel = None@@ -597,22 +575,21 @@ def run():print(f"Failed to start Cloudflare tunnel: {e}")print("Continuing without tunnel...")+ # initialize contexts from persisted chats+ persist_chat.load_tmp_chats()+try:# Run Flask appapp.run(- request_handler=NoRequestLoggingWSGIRequestHandler,- port=port,- host=host+ request_handler=NoRequestLoggingWSGIRequestHandler, port=port, host=host)finally:# Clean up tunnel if it was startedif tunnel:tunnel.stop()+# run the internal serverif __name__ == "__main__":-- parser.add_argument("--port", type=int, default=0, help="Web UI port")- parser.add_argument("--host", type=str, default=0, help="Web UI host")-+ runtime.initialize()run()diff --git a/webui/settings.css b/webui/settings.cssindex 50c8af021..d38a5212b 100644--- a/webui/settings.css+++ b/webui/settings.css@@ -102,7 +102,7 @@ select {.field {display: grid;margin-block-start: 1rem;- grid-template-columns: 250px 1fr;+ grid-template-columns: 60% 1fr;align-items: center;}@@ -113,6 +113,7 @@ select {.field-label {display: flex;flex-direction: column;+ padding-right: 0.5em;}.field-title {diff --git a/webui/speech.js b/webui/speech.jsindex 69c545fb7..cbf8be179 100644--- a/webui/speech.js+++ b/webui/speech.js@@ -251,12 +251,13 @@ class MicrophoneInput {}const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });- const audioUrl = URL.createObjectURL(audioBlob);+ const base64 = await this.convertBlobToBase64Wav(audioBlob)try {- const samplingRate = 16000;- const audioData = await read_audio(audioUrl, samplingRate);- const result = await this.transcriber(audioData);++ const result = await sendJsonData('/transcribe', { audio: base64 })++const text = this.filterResult(result.text || "")if (text) {@@ -267,12 +268,29 @@ class MicrophoneInput {console.error('Transcription error:', error);toast('Transcription failed.', 'error');} finally {- URL.revokeObjectURL(audioUrl);this.audioChunks = [];this.status = Status.LISTENING;}}+ convertBlobToBase64Wav(audioBlob) {+ return new Promise((resolve, reject) => {+ const reader = new FileReader();++ // Read the Blob as a Data URL+ reader.onloadend = () => {+ const base64Data = reader.result.split(",")[1]; // Extract Base64 data+ resolve(base64Data);+ };++ reader.onerror = (error) => {+ reject(error);+ };++ reader.readAsDataURL(audioBlob); // Start reading the Blob+ });+ }+filterResult(text) {text = text.trim()let ok = false@@ -370,12 +388,45 @@ class Speech {// Remove emojis and create a new utterancetext = this.stripEmojis(text);+ text = this.replaceURLs(text);+ text = this.replaceGuids(text);this.utterance = new SpeechSynthesisUtterance(text);// Speak the new utterancethis.synth.speak(this.utterance);}+ replaceURLs(text) {+ const urlRegex = /(\b(https?|ftp|file):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])|(\b(www\.)[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])|(\b[-A-Z0-9+&@#\/%?=~_|!:,.;]*\.(?:[A-Z]{2,})[-A-Z0-9+&@#\/%?=~_|])/ig; return text.replace(urlRegex, (url) => {+ let text = url+ // if contains ://, split by it+ if (text.includes('://')) text = text.split('://')[1];+ // if contains /, split by it+ if (text.includes('/')) text = text.split('/')[0];++ // if contains ., split by it+ if (text.includes('.')) {+ const doms = text.split('.')+ //up to last two+ return doms[doms.length - 2] + '.' + doms[doms.length - 1]+ } else {+ return text+ }+ });+ }++ replaceGuids(text) {+ const guidRegex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g;+ return text.replace(guidRegex, '');+ }++ replaceNonText(text) {+ const nonTextRegex = /\w[^\w\s]*\w(?=\s|$)|[^\w\s]+/g;+ return text.replace(nonTextRegex, (match) => {+ return ``;+ });+ }+stop() {if (this.isSpeaking()) {this.synth.cancel();diff --git a/webui/speech_browser.js b/webui/speech_browser.jsnew file mode 100644index 000000000..4ab46ab80--- /dev/null+++ b/webui/speech_browser.js@@ -0,0 +1,394 @@+import { pipeline, read_audio } from './[email protected]';+import { updateChatInput, sendMessage } from './index.js';++const microphoneButton = document.getElementById('microphone-button');+let microphoneInput = null;+let isProcessingClick = false;++const Status = {+ INACTIVE: 'inactive',+ ACTIVATING: 'activating',+ LISTENING: 'listening',+ RECORDING: 'recording',+ WAITING: 'waiting',+ PROCESSING: 'processing'+};++class MicrophoneInput {+ constructor(updateCallback, options = {}) {+ this.mediaRecorder = null;+ this.audioChunks = [];+ this.lastChunk = [];+ this.updateCallback = updateCallback;+ this.messageSent = false;++ // Audio analysis properties+ this.audioContext = null;+ this.mediaStreamSource = null;+ this.analyserNode = null;+ this._status = Status.INACTIVE;++ // Timing properties+ this.lastAudioTime = null;+ this.waitingTimer = null;+ this.silenceStartTime = null;+ this.hasStartedRecording = false;+ this.analysisFrame = null;++ this.options = {+ modelSize: 'tiny',+ language: 'en',+ silenceThreshold: 0.15,+ silenceDuration: 1000,+ waitingTimeout: 2000,+ minSpeechDuration: 500,+ ...options+ };+ }++ get status() {+ return this._status;+ }++ set status(newStatus) {+ if (this._status === newStatus) return;++ const oldStatus = this._status;+ this._status = newStatus;+ console.log(`Mic status changed from ${oldStatus} to ${newStatus}`);++ // Update UI+ microphoneButton.classList.remove(`mic-${oldStatus.toLowerCase()}`);+ microphoneButton.classList.add(`mic-${newStatus.toLowerCase()}`);+ microphoneButton.setAttribute('data-status', newStatus);++ // Handle state-specific behaviors+ this.handleStatusChange(oldStatus, newStatus);+ }++ handleStatusChange(oldStatus, newStatus) {++ //last chunk kept only for transition to recording status+ if (newStatus != Status.RECORDING) { this.lastChunk = null; }++ switch (newStatus) {+ case Status.INACTIVE:+ this.handleInactiveState();+ break;+ case Status.LISTENING:+ this.handleListeningState();+ break;+ case Status.RECORDING:+ this.handleRecordingState();+ break;+ case Status.WAITING:+ this.handleWaitingState();+ break;+ case Status.PROCESSING:+ this.handleProcessingState();+ break;+ }+ }++ handleInactiveState() {+ this.stopRecording();+ this.stopAudioAnalysis();+ if (this.waitingTimer) {+ clearTimeout(this.waitingTimer);+ this.waitingTimer = null;+ }+ }++ handleListeningState() {+ this.stopRecording();+ this.audioChunks = [];+ this.hasStartedRecording = false;+ this.silenceStartTime = null;+ this.lastAudioTime = null;+ this.messageSent = false;+ this.startAudioAnalysis();+ }++ handleRecordingState() {+ if (!this.hasStartedRecording && this.mediaRecorder.state !== 'recording') {+ this.hasStartedRecording = true;+ this.mediaRecorder.start(1000);+ console.log('Speech started');+ }+ if (this.waitingTimer) {+ clearTimeout(this.waitingTimer);+ this.waitingTimer = null;+ }+ }++ handleWaitingState() {+ // Don't stop recording during waiting state+ this.waitingTimer = setTimeout(() => {+ if (this.status === Status.WAITING) {+ this.status = Status.PROCESSING;+ }+ }, this.options.waitingTimeout);+ }++ handleProcessingState() {+ this.stopRecording();+ this.process();+ }++ stopRecording() {+ if (this.mediaRecorder?.state === 'recording') {+ this.mediaRecorder.stop();+ this.hasStartedRecording = false;+ }+ }++ async initialize() {+ try {+ this.transcriber = await pipeline(+ 'automatic-speech-recognition',+ `Xenova/whisper-${this.options.modelSize}.${this.options.language}`+ );++ const stream = await navigator.mediaDevices.getUserMedia({+ audio: {+ echoCancellation: true,+ noiseSuppression: true,+ channelCount: 1+ }+ });++ this.mediaRecorder = new MediaRecorder(stream);+ this.mediaRecorder.ondataavailable = (event) => {+ if (event.data.size > 0 &&+ (this.status === Status.RECORDING || this.status === Status.WAITING)) {+ if (this.lastChunk) {+ this.audioChunks.push(this.lastChunk);+ this.lastChunk = null;+ }+ this.audioChunks.push(event.data);+ console.log('Audio chunk received, total chunks:', this.audioChunks.length);+ }+ else if (this.status === Status.LISTENING) {+ this.lastChunk = event.data;+ }+ };++ this.setupAudioAnalysis(stream);+ return true;+ } catch (error) {++ console.error('Microphone initialization error:', error);+ toast('Failed to access microphone. Please check permissions.', 'error');+ return false;+ }+ }++ setupAudioAnalysis(stream) {+ this.audioContext = new (window.AudioContext || window.webkitAudioContext)();+ this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);+ this.analyserNode = this.audioContext.createAnalyser();+ this.analyserNode.fftSize = 2048;+ this.analyserNode.minDecibels = -90;+ this.analyserNode.maxDecibels = -10;+ this.analyserNode.smoothingTimeConstant = 0.85;+ this.mediaStreamSource.connect(this.analyserNode);+ }+++ startAudioAnalysis() {+ const analyzeFrame = () => {+ if (this.status === Status.INACTIVE) return;++ const dataArray = new Uint8Array(this.analyserNode.fftSize);+ this.analyserNode.getByteTimeDomainData(dataArray);++ // Calculate RMS volume+ let sum = 0;+ for (let i = 0; i < dataArray.length; i++) {+ const amplitude = (dataArray[i] - 128) / 128;+ sum += amplitude * amplitude;+ }+ const rms = Math.sqrt(sum / dataArray.length);++ const now = Date.now();++ // Update status based on audio level+ if (rms > this.options.silenceThreshold) {+ this.lastAudioTime = now;+ this.silenceStartTime = null;++ if (this.status === Status.LISTENING || this.status === Status.WAITING) {+ if (!speech.isSpeaking()) // TODO? a better way to ignore agent's voice?+ this.status = Status.RECORDING;+ }+ } else if (this.status === Status.RECORDING) {+ if (!this.silenceStartTime) {+ this.silenceStartTime = now;+ }++ const silenceDuration = now - this.silenceStartTime;+ if (silenceDuration >= this.options.silenceDuration) {+ this.status = Status.WAITING;+ }+ }++ this.analysisFrame = requestAnimationFrame(analyzeFrame);+ };++ this.analysisFrame = requestAnimationFrame(analyzeFrame);+ }++ stopAudioAnalysis() {+ if (this.analysisFrame) {+ cancelAnimationFrame(this.analysisFrame);+ this.analysisFrame = null;+ }+ }++ async process() {+ if (this.audioChunks.length === 0) {+ this.status = Status.LISTENING;+ return;+ }++ const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });+ const audioUrl = URL.createObjectURL(audioBlob);++++ try {+ const samplingRate = 16000;+ const audioData = await read_audio(audioUrl, samplingRate);+ const result = await this.transcriber(audioData);+ const text = this.filterResult(result.text || "")++ if (text) {+ console.log('Transcription:', result.text);+ await this.updateCallback(result.text, true);+ }+ } catch (error) {+ console.error('Transcription error:', error);+ toast('Transcription failed.', 'error');+ } finally {+ URL.revokeObjectURL(audioUrl);+ this.audioChunks = [];+ this.status = Status.LISTENING;+ }+ }++ filterResult(text) {+ text = text.trim()+ let ok = false+ while (!ok) {+ if (!text) break+ if (text[0] === '{' && text[text.length - 1] === '}') break+ if (text[0] === '(' && text[text.length - 1] === ')') break+ if (text[0] === '[' && text[text.length - 1] === ']') break+ ok = true+ }+ if (ok) return text+ else console.log(`Discarding transcription: ${text}`)+ }+}++++// Initialize and handle click events+async function initializeMicrophoneInput() {+ microphoneInput = new MicrophoneInput(+ async (text, isFinal) => {+ if (isFinal) {+ updateChatInput(text);+ if (!microphoneInput.messageSent) {+ microphoneInput.messageSent = true;+ await sendMessage();+ }+ }+ },+ {+ modelSize: 'tiny',+ language: 'en',+ silenceThreshold: 0.07,+ silenceDuration: 1000,+ waitingTimeout: 1500+ }+ );+ microphoneInput.status = Status.ACTIVATING;++ return await microphoneInput.initialize();+}++microphoneButton.addEventListener('click', async () => {+ if (isProcessingClick) return;+ isProcessingClick = true;++ const hasPermission = await requestMicrophonePermission();+ if (!hasPermission) return;++ try {+ if (!microphoneInput && !await initializeMicrophoneInput()) {+ return;+ }++ // Simply toggle between INACTIVE and LISTENING states+ microphoneInput.status =+ (microphoneInput.status === Status.INACTIVE || microphoneInput.status === Status.ACTIVATING) ? Status.LISTENING : Status.INACTIVE;+ } finally {+ setTimeout(() => {+ isProcessingClick = false;+ }, 300);+ }+});++// Some error handling for microphone input+async function requestMicrophonePermission() {+ try {+ await navigator.mediaDevices.getUserMedia({ audio: true });+ return true;+ } catch (err) {+ console.error('Error accessing microphone:', err);+ toast('Microphone access denied. Please enable microphone access in your browser settings.', 'error');+ return false;+ }+}+++class Speech {+ constructor() {+ this.synth = window.speechSynthesis;+ this.utterance = null;+ }++ stripEmojis(str) {+ return str+ .replace(/([\u2700-\u27BF]|[\uE000-\uF8FF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDFFF]|[\u2011-\u26FF]|\uD83E[\uDD10-\uDDFF])/g, '')+ .replace(/\s+/g, ' ')+ .trim();+ }++ speak(text) {+ console.log('Speaking:', text);+ // Stop any current utterance+ this.stop();++ // Remove emojis and create a new utterance+ text = this.stripEmojis(text);+ this.utterance = new SpeechSynthesisUtterance(text);++ // Speak the new utterance+ this.synth.speak(this.utterance);+ }++ stop() {+ if (this.isSpeaking()) {+ this.synth.cancel();+ }+ }++ isSpeaking() {+ return this.synth?.speaking || false;+ }+}++export const speech = new Speech();+window.speech = speech\ No newline at end of file