Skip to content

Agent Zero AI framework

Squashed commit of the following:

frdel9c7339042f1c
diff --git a/.gitignore b/.gitignore
index e10c1fd6b..5a668772b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -65,4 +65,4 @@ instruments/**/*.*
# Ignore all contents of the directory "bin"
bin/*
# But do not ignore the directory itself
-!bin/.gitkeep
\ No newline at end of file
+!bin/.gitkeep
diff --git a/.vscode/launch.json b/.vscode/launch.json
index 37ab9f096..883a8edbc 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -1,29 +1,29 @@
{
- "version": "0.2.0",
- "configurations": [
- {
- "name": "Debug run_ui.py",
- "type": "debugpy",
- "request": "launch",
- "program": "./run_ui.py",
- "console": "integratedTerminal",
- "args": ["-Xfrozen_modules=off"]
- },
- {
- "name": "Debug run_cli.py",
- "type": "debugpy",
- "request": "launch",
- "program": "./run_cli.py",
- "console": "integratedTerminal",
- "args": ["-Xfrozen_modules=off"]
- },
- {
- "name": "Debug current file",
- "type": "debugpy",
- "request": "launch",
- "program": "${file}",
- "console": "integratedTerminal",
- "args": ["-Xfrozen_modules=off"]
- }
- ]
-}
\ No newline at end of file
+ "version": "0.2.0",
+ "configurations": [
+ {
+ "name": "Debug run_ui.py",
+ "type": "debugpy",
+ "request": "launch",
+ "program": "./run_ui.py",
+ "console": "integratedTerminal",
+ "args": ["--development=true", "-Xfrozen_modules=off"]
+ },
+ {
+ "name": "Debug run_cli.py",
+ "type": "debugpy",
+ "request": "launch",
+ "program": "./run_cli.py",
+ "console": "integratedTerminal",
+ "args": ["--development=true", "-Xfrozen_modules=off"]
+ },
+ {
+ "name": "Debug current file",
+ "type": "debugpy",
+ "request": "launch",
+ "program": "${file}",
+ "console": "integratedTerminal",
+ "args": ["--development=true", "-Xfrozen_modules=off"]
+ }
+ ]
+}
diff --git a/agent.py b/agent.py
index 2bac75793..f3d8a4f60 100644
--- a/agent.py
+++ b/agent.py
@@ -457,7 +457,7 @@ class Agent:
printer = PrintStyle(italic=True, font_color="orange", padding=False)
def log_callback(content):
- printer.print(content)
+ printer.stream(content)
log_item.stream(content=content)
summary = await self.call_utility_llm(
diff --git a/docker/docker_manager_macos.sh b/docker/docker_manager_mac_linux.sh
similarity index 100%
rename from docker/docker_manager_macos.sh
rename to docker/docker_manager_mac_linux.sh
diff --git a/docker/run/Dockerfile b/docker/run/Dockerfile
index 5049a5d3e..c38b778c6 100644
--- a/docker/run/Dockerfile
+++ b/docker/run/Dockerfile
@@ -21,9 +21,6 @@ RUN apt-get update && apt-get install -y \
# Cleanup package list
RUN rm -rf /var/lib/apt/lists/*
-# Clone Agent Zero repository
-RUN git clone --branch development https://github.com/frdel/agent-zero.git /git/agent-zero
-
# Set up SSH
RUN mkdir /var/run/sshd && \
echo 'root:toor' | chpasswd && \
@@ -33,12 +30,8 @@ RUN mkdir /var/run/sshd && \
ENV VIRTUAL_ENV=/opt/venv
RUN python3 -m venv $VIRTUAL_ENV
-# Copy the script to ensure .bashrc is in the root directory
-COPY initialize.sh /usr/local/bin/initialize.sh
-RUN chmod +x /usr/local/bin/initialize.sh
-
-# Copy contents of filesystem directory to /fs
-COPY ./fs/ /fs
+# Copy contents of filesystem directory to /
+COPY ./fs/ /
# Ensure the virtual environment and pip setup
RUN $VIRTUAL_ENV/bin/pip install --upgrade pip
@@ -48,13 +41,27 @@ RUN $VIRTUAL_ENV/bin/pip install \
ipython \
requests
+# Clone Agent Zero repository
+RUN git clone --branch development https://github.com/frdel/agent-zero.git /git/agent-zero
+
# Install A0 python packages
RUN $VIRTUAL_ENV/bin/pip install -r /git/agent-zero/requirements.txt
# Preload A0
RUN $VIRTUAL_ENV/bin/python /git/agent-zero/preload.py
+
+
+# Clone additional git repos
+RUN git clone https://github.com/searxng/searxng.git /git/searxng
+
+
+# install additional software
+RUN bash /ins/install_searxng.sh
+
+
+
# Expose ports
EXPOSE 22 80
-# Init .bashrc
-CMD ["/usr/local/bin/initialize.sh"]
\ No newline at end of file
+# initialize runtime
+CMD ["/bin/bash", "/exe/initialize.sh"]
\ No newline at end of file
diff --git a/docker/run/build.txt b/docker/run/build.txt
index 605762818..d5d52dd2d 100644
--- a/docker/run/build.txt
+++ b/docker/run/build.txt
@@ -3,4 +3,4 @@ docker build -t agent-zero-run:latest .
# dockerhub
docker login
-docker buildx build --platform linux/amd64,linux/arm64 -t frdel/agent-zero-run:latest --push .
\ No newline at end of file
+docker buildx build --platform linux/amd64,linux/arm64 -t frdel/agent-zero-run:testing --push .
\ No newline at end of file
diff --git a/docker/run/fs/etc/searxng/settings.yml b/docker/run/fs/etc/searxng/settings.yml
new file mode 100644
index 000000000..7b9f3b0d1
--- /dev/null
+++ b/docker/run/fs/etc/searxng/settings.yml
@@ -0,0 +1,77 @@
+# SearXNG settings
+
+use_default_settings: true
+
+general:
+ debug: false
+ instance_name: "SearXNG"
+
+search:
+ safe_search: 2
+ autocomplete: 'duckduckgo'
+ formats:
+ - html
+ - json
+
+server:
+ # Is overwritten by ${SEARXNG_SECRET}
+ secret_key: "1e91dce279fdef87eb7b6f43029c206b"
+ limiter: true
+ image_proxy: true
+ # public URL of the instance, to ensure correct inbound links. Is overwritten
+ # by ${SEARXNG_URL}.
+ # base_url: http://example.com/location
+
+redis:
+ # URL to connect redis database. Is overwritten by ${SEARXNG_REDIS_URL}.
+ url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
+
+ui:
+ static_use_hash: true
+
+# preferences:
+# lock:
+# - autocomplete
+# - method
+
+enabled_plugins:
+ - 'Hash plugin'
+ - 'Self Informations'
+ - 'Tracker URL remover'
+ - 'Ahmia blacklist'
+ # - 'Hostnames plugin' # see 'hostnames' configuration below
+ # - 'Open Access DOI rewrite'
+
+# plugins:
+# - only_show_green_results
+
+# hostnames:
+# replace:
+# '(.*\.)?youtube\.com$': 'invidious.example.com'
+# '(.*\.)?youtu\.be$': 'invidious.example.com'
+# remove:
+# - '(.*\.)?facebook.com$'
+# low_priority:
+# - '(.*\.)?google\.com$'
+# high_priority:
+# - '(.*\.)?wikipedia.org$'
+
+engines:
+
+# - name: fdroid
+# disabled: false
+#
+# - name: apk mirror
+# disabled: false
+#
+# - name: mediathekviewweb
+# categories: TV
+# disabled: false
+#
+# - name: invidious
+# disabled: false
+# base_url:
+# - https://invidious.snopyta.org
+# - https://invidious.tiekoetter.com
+# - https://invidio.xamh.de
+# - https://inv.riverside.rocks
\ No newline at end of file
diff --git a/docker/run/initialize.sh b/docker/run/fs/exe/initialize.sh
similarity index 61%
rename from docker/run/initialize.sh
rename to docker/run/fs/exe/initialize.sh
index 1f9b0db5b..993379053 100644
--- a/docker/run/initialize.sh
+++ b/docker/run/fs/exe/initialize.sh
@@ -1,18 +1,21 @@
#!/bin/bash
-# Copy all contents from /fs to root directory (/) without overwriting
-cp -rn --no-preserve=ownership,mode /fs/* /
+# Copy all contents from persistent /per to root directory (/) without overwriting
+cp -r --no-preserve=ownership,mode /per/* /
# allow execution of /root/.bashrc and /root/.profile
chmod 444 /root/.bashrc
chmod 444 /root/.profile
# update package list to save time later
-apt-get update
+apt-get update &
# Start SSH service in background
/usr/sbin/sshd -D &
+# Start searxng server in background
+sudo -H -u searxng -i bash /exe/run_searxng.sh &
+
# Start A0 and restart on exit
bash /exe/run_A0.sh
if [ $? -ne 0 ]; then
diff --git a/docker/run/fs/exe/run_A0.sh b/docker/run/fs/exe/run_A0.sh
index 266c6459c..0da64a97f 100644
--- a/docker/run/fs/exe/run_A0.sh
+++ b/docker/run/fs/exe/run_A0.sh
@@ -34,14 +34,14 @@ while true; do
echo "Starting A0..."
python /a0/run_ui.py \
- --port 80 \
- --host "0.0.0.0" \
- --code_exec_docker_enabled False \
- --code_exec_ssh_enabled True \
- --code_exec_ssh_addr "localhost" \
- --code_exec_ssh_port 22 \
- --code_exec_ssh_user "root" \
- --code_exec_ssh_pass "toor"
+ --port=80 \
+ --host="0.0.0.0" \
+ --code_exec_docker_enabled=false \
+ --code_exec_ssh_enabled=true \
+ --code_exec_ssh_addr="localhost" \
+ --code_exec_ssh_port=22 \
+ --code_exec_ssh_user="root" \
+ --code_exec_ssh_pass="toor"
# Check the exit status
if [ $? -ne 0 ]; then
diff --git a/docker/run/fs/exe/run_searxng.sh b/docker/run/fs/exe/run_searxng.sh
new file mode 100644
index 000000000..880314f29
--- /dev/null
+++ b/docker/run/fs/exe/run_searxng.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# start webapp
+sudo -H -u searxng -i
+cd /usr/local/searxng/searxng-src
+export SEARXNG_SETTINGS_PATH="/etc/searxng/settings.yml"
+python searx/webapp.py
\ No newline at end of file
diff --git a/docker/run/fs/ins/install_searxng.sh b/docker/run/fs/ins/install_searxng.sh
new file mode 100644
index 000000000..ac76a30da
--- /dev/null
+++ b/docker/run/fs/ins/install_searxng.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+sudo -H apt-get install -y \
+ python3-dev python3-babel python3-venv \
+ uwsgi uwsgi-plugin-python3 \
+ git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev
+
+sudo -H useradd --shell /bin/bash --system \
+ --home-dir "/usr/local/searxng" \
+ --comment 'Privacy-respecting metasearch engine' \
+ searxng
+
+sudo -H mkdir "/usr/local/searxng"
+sudo -H chown -R "searxng:searxng" "/usr/local/searxng"
+
+# Start a new shell from new created user and clone SearXNG:
+sudo -H -u searxng -i bash /ins/install_searxng_user.sh
diff --git a/docker/run/fs/ins/install_searxng_user.sh b/docker/run/fs/ins/install_searxng_user.sh
new file mode 100644
index 000000000..0bc2132bf
--- /dev/null
+++ b/docker/run/fs/ins/install_searxng_user.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# clone SearXNG repo
+# git clone "https://github.com/searxng/searxng" \
+# "/usr/local/searxng/searxng-src"
+cp -r "/git/searxng" "/usr/local/searxng/searxng-src"
+
+# create virtualenv:
+python3 -m venv "/usr/local/searxng/searx-pyenv"
+
+# make it default
+echo ". /usr/local/searxng/searx-pyenv/bin/activate" \
+ >> "/usr/local/searxng/.profile"
+
+# activate venv
+source "/usr/local/searxng/searx-pyenv/bin/activate"
+
+# update pip's boilerplate
+pip install -U pip
+pip install -U setuptools
+pip install -U wheel
+pip install -U pyyaml
+
+# jump to SearXNG's working tree and install SearXNG into virtualenv
+cd "/usr/local/searxng/searxng-src"
+pip install --use-pep517 --no-build-isolation -e .
diff --git a/docker/run/fs/root/.bashrc b/docker/run/fs/per/root/.bashrc
similarity index 100%
rename from docker/run/fs/root/.bashrc
rename to docker/run/fs/per/root/.bashrc
diff --git a/docker/run/fs/root/.profile b/docker/run/fs/per/root/.profile
similarity index 100%
rename from docker/run/fs/root/.profile
rename to docker/run/fs/per/root/.profile
diff --git a/initialize.py b/initialize.py
index e8b6d8611..24e84aa67 100644
--- a/initialize.py
+++ b/initialize.py
@@ -1,15 +1,12 @@
import models
from agent import AgentConfig
-from python.helpers import files, settings
+from python.helpers import files, runtime, settings
-global_kwargs = {}
-
-def set_global_kwargs(**kwargs):
- global global_kwargs
- global_kwargs = kwargs
def initialize():
-
+
+ current_settings = settings.get_settings()
+
# main chat model used by agents (smarter, more accurate)
# chat_llm = models.get_openai_chat(model_name="gpt-4o-mini", temperature=0)
# chat_llm = models.get_ollama_chat(model_name="llama3.2:3b-instruct-fp16", temperature=0)
@@ -21,44 +18,50 @@ def initialize():
# chat_llm = models.get_mistral_chat(model_name="mistral-small-latest", temperature=0)
# chat_llm = models.get_groq_chat(model_name="llama-3.2-90b-text-preview", temperature=0)
# chat_llm = models.get_sambanova_chat(model_name="Meta-Llama-3.1-70B-Instruct-8k", temperature=0)
- chat_llm = settings.get_chat_model() # chat model from user settings
+ chat_llm = settings.get_chat_model(
+ current_settings
+ ) # chat model from user settings
# utility model used for helper functions (cheaper, faster)
# utility_llm = chat_llm
- utility_llm = settings.get_utility_model() # utility model from user settings
+ utility_llm = settings.get_utility_model(
+ current_settings
+ ) # utility model from user settings
# embedding model used for memory
# embedding_llm = models.get_openai_embedding(model_name="text-embedding-3-small")
# embedding_llm = models.get_ollama_embedding(model_name="nomic-embed-text")
# embedding_llm = models.get_huggingface_embedding(model_name="sentence-transformers/all-MiniLM-L6-v2")
# embedding_llm = models.get_lmstudio_embedding(model_name="nomic-ai/nomic-embed-text-v1.5-GGUF")
- embedding_llm = settings.get_embedding_model() # embedding model from user settings
-
+ embedding_llm = settings.get_embedding_model(
+ current_settings
+ ) # embedding model from user settings
+
# agent configuration
config = AgentConfig(
- chat_model = chat_llm,
- utility_model = utility_llm,
- embeddings_model = embedding_llm,
- # prompts_subdir = "default",
- # memory_subdir = "",
- knowledge_subdirs = ["default","custom"],
+ chat_model=chat_llm,
+ utility_model=utility_llm,
+ embeddings_model=embedding_llm,
+ prompts_subdir=current_settings["agent_prompts_subdir"],
+ memory_subdir=current_settings["agent_memory_subdir"],
+ knowledge_subdirs=["default", current_settings["agent_knowledge_subdir"]],
# rate_limit_seconds = 60,
- rate_limit_requests = 30,
+ rate_limit_requests=30,
# rate_limit_input_tokens = 0,
# rate_limit_output_tokens = 0,
# msgs_keep_max = 25,
# msgs_keep_start = 5,
# msgs_keep_end = 10,
- max_tool_response_length = 3000,
+ max_tool_response_length=3000,
# response_timeout_seconds = 60,
# code_exec_docker_enabled = True,
# code_exec_docker_name = "agent-zero-exe",
# code_exec_docker_image = "frdel/agent-zero-exe:latest",
# code_exec_docker_ports = { "22/tcp": 50022 }
- # code_exec_docker_volumes = {
- # files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"},
- # files.get_abs_path("instruments"): {"bind": "/instruments", "mode": "rw"},
- # },
+ # code_exec_docker_volumes = {
+ # files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"},
+ # files.get_abs_path("instruments"): {"bind": "/instruments", "mode": "rw"},
+ # },
# code_exec_ssh_enabled = True,
# code_exec_ssh_addr = "localhost",
# code_exec_ssh_port = 50022,
@@ -68,9 +71,27 @@ def initialize():
)
# update config with kwargs
- for key, value in global_kwargs.items():
+ for key, value in runtime.args.items():
if hasattr(config, key):
+ # conversion based on type of config[key]
+ if isinstance(getattr(config, key), bool):
+ value = value.lower().strip() == "true"
+ print("bool", value)
+ elif isinstance(getattr(config, key), int):
+ value = int(value)
+ print("int", value)
+ elif isinstance(getattr(config, key), float):
+ value = float(value)
+ print("float", value)
+ elif isinstance(getattr(config, key), str):
+ value = str(value)
+ print("str", value)
+ else:
+ raise Exception(
+ f"Unsupported argument type of '{key}': {type(getattr(config, key))}"
+ )
+
setattr(config, key, value)
-
+
# return config object
return config
diff --git a/instruments/default/yt_download/yt_download.md b/instruments/default/yt_download/yt_download.md
index 25f09ffd6..0316d0583 100644
--- a/instruments/default/yt_download/yt_download.md
+++ b/instruments/default/yt_download/yt_download.md
@@ -2,5 +2,5 @@
Download a YouTube video
# Solution
1. If folder is specified, cd to it
-2. Run instrument "bash /instruments/default/yt_download/yt_download.sh <url>" with your video URL
+2. Run instrument "bash /a0/instruments/default/yt_download/yt_download.sh <url>" with your video URL
3. Wait for the terminal to finish
\ No newline at end of file
diff --git a/models.py b/models.py
index ed194bae3..85b0805db 100644
--- a/models.py
+++ b/models.py
@@ -58,7 +58,11 @@ class ModelProvider(Enum):
# Utility function to get API keys from environment variables
def get_api_key(service):
- return dotenv.get_dotenv_value(f"API_KEY_{service.upper()}") or dotenv.get_dotenv_value(f"{service.upper()}_API_KEY") or "None"
+ return (
+ dotenv.get_dotenv_value(f"API_KEY_{service.upper()}")
+ or dotenv.get_dotenv_value(f"{service.upper()}_API_KEY")
+ or "None"
+ )
def get_model(type: ModelType, provider: ModelProvider, name: str, **kwargs):
@@ -71,10 +75,12 @@ def get_model(type: ModelType, provider: ModelProvider, name: str, **kwargs):
def get_ollama_chat(
model_name: str,
temperature=DEFAULT_TEMPERATURE,
- base_url=dotenv.get_dotenv_value("OLLAMA_BASE_URL") or "http://127.0.0.1:11434",
+ base_url=None,
num_ctx=8192,
**kwargs,
):
+ if not base_url:
+ base_url = dotenv.get_dotenv_value("OLLAMA_BASE_URL") or "http://127.0.0.1:11434"
return ChatOllama(
model=model_name,
temperature=temperature,
@@ -87,9 +93,11 @@ def get_ollama_chat(
def get_ollama_embedding(
model_name: str,
temperature=DEFAULT_TEMPERATURE,
- base_url=dotenv.get_dotenv_value("OLLAMA_BASE_URL") or "http://127.0.0.1:11434",
+ base_url=None,
**kwargs,
):
+ if not base_url:
+ base_url = dotenv.get_dotenv_value("OLLAMA_BASE_URL") or "http://127.0.0.1:11434"
return OllamaEmbeddings(
model=model_name, temperature=temperature, base_url=base_url, **kwargs
)
@@ -98,13 +106,13 @@ def get_ollama_embedding(
# HuggingFace models
def get_huggingface_chat(
model_name: str,
- api_key=get_api_key("huggingface"),
+ api_key=None,
temperature=DEFAULT_TEMPERATURE,
**kwargs,
):
# different naming convention here
if not api_key:
- api_key = os.environ["HUGGINGFACEHUB_API_TOKEN"]
+ api_key = get_api_key("huggingface") or os.environ["HUGGINGFACEHUB_API_TOKEN"]
# Initialize the HuggingFaceEndpoint with the specified model and parameters
llm = HuggingFaceEndpoint(
@@ -127,7 +135,8 @@ def get_huggingface_embedding(model_name: str, **kwargs):
def get_lmstudio_chat(
model_name: str,
temperature=DEFAULT_TEMPERATURE,
- base_url=dotenv.get_dotenv_value("LM_STUDIO_BASE_URL") or "http://127.0.0.1:1234/v1",
+ base_url=dotenv.get_dotenv_value("LM_STUDIO_BASE_URL")
+ or "http://127.0.0.1:1234/v1",
**kwargs,
):
return ChatOpenAI(model_name=model_name, base_url=base_url, temperature=temperature, api_key="none", **kwargs) # type: ignore
@@ -135,7 +144,8 @@ def get_lmstudio_chat(
def get_lmstudio_embedding(
model_name: str,
- base_url=dotenv.get_dotenv_value("LM_STUDIO_BASE_URL") or "http://127.0.0.1:1234/v1",
+ base_url=dotenv.get_dotenv_value("LM_STUDIO_BASE_URL")
+ or "http://127.0.0.1:1234/v1",
**kwargs,
):
return OpenAIEmbeddings(model=model_name, api_key="none", base_url=base_url, check_embedding_ctx_length=False, **kwargs) # type: ignore
@@ -144,151 +154,172 @@ def get_lmstudio_embedding(
# Anthropic models
def get_anthropic_chat(
model_name: str,
- api_key=get_api_key("anthropic"),
+ api_key=None,
temperature=DEFAULT_TEMPERATURE,
**kwargs,
):
+ if not api_key:
+ api_key = get_api_key("anthropic")
return ChatAnthropic(model_name=model_name, temperature=temperature, api_key=api_key, **kwargs) # type: ignore
# right now anthropic does not have embedding models, but that might change
def get_anthropic_embedding(
model_name: str,
- api_key=get_api_key("anthropic"),
+ api_key=None,
**kwargs,
):
+ if not api_key:
+ api_key = get_api_key("anthropic")
return OpenAIEmbeddings(model=model_name, api_key=api_key, **kwargs) # type: ignore
# OpenAI models
def get_openai_chat(
model_name: str,
- api_key=get_api_key("openai"),
+ api_key=None,
temperature=DEFAULT_TEMPERATURE,
**kwargs,
):
+ if not api_key:
+ api_key = get_api_key("openai")
return ChatOpenAI(model_name=model_name, temperature=temperature, api_key=api_key, **kwargs) # type: ignore
-def get_openai_instruct(
- model_name: str,
- api_key=get_api_key("openai"),
- temperature=DEFAULT_TEMPERATURE,
- **kwargs,
-):
- return OpenAI(model=model_name, temperature=temperature, api_key=api_key, **kwargs) # type: ignore
-
-
-def get_openai_embedding(model_name: str, api_key=get_api_key("openai"), **kwargs):
+def get_openai_embedding(model_name: str, api_key=None, **kwargs):
+ if not api_key:
+ api_key = get_api_key("openai")
return OpenAIEmbeddings(model=model_name, api_key=api_key, **kwargs) # type: ignore
def get_azure_openai_chat(
deployment_name: str,
- api_key=get_api_key("openai_azure"),
+ api_key=None,
temperature=DEFAULT_TEMPERATURE,
- azure_endpoint=dotenv.get_dotenv_value("OPENAI_AZURE_ENDPOINT"),
+ azure_endpoint=None,
**kwargs,
):
+ if not api_key:
+ api_key = get_api_key("openai_azure")
+ if not azure_endpoint:
+ azure_endpoint = dotenv.get_dotenv_value("OPENAI_AZURE_ENDPOINT")
return AzureChatOpenAI(deployment_name=deployment_name, temperature=temperature, api_key=api_key, azure_endpoint=azure_endpoint, **kwargs) # type: ignore
-def get_azure_openai_instruct(
- deployment_name: str,
- api_key=get_api_key("openai_azure"),
- temperature=DEFAULT_TEMPERATURE,
- azure_endpoint=dotenv.get_dotenv_value("OPENAI_AZURE_ENDPOINT"),
- **kwargs,
-):
- return AzureOpenAI(deployment_name=deployment_name, temperature=temperature, api_key=api_key, azure_endpoint=azure_endpoint, **kwargs) # type: ignore
-
-
def get_azure_openai_embedding(
deployment_name: str,
- api_key=get_api_key("openai_azure"),
- azure_endpoint=dotenv.get_dotenv_value("OPENAI_AZURE_ENDPOINT"),
+ api_key=None,
+ azure_endpoint=None,
**kwargs,
):
+ if not api_key:
+ api_key = get_api_key("openai_azure")
+ if not azure_endpoint:
+ azure_endpoint = dotenv.get_dotenv_value("OPENAI_AZURE_ENDPOINT")
return AzureOpenAIEmbeddings(deployment_name=deployment_name, api_key=api_key, azure_endpoint=azure_endpoint, **kwargs) # type: ignore
# Google models
def get_google_chat(
model_name: str,
- api_key=get_api_key("google"),
+ api_key=None,
temperature=DEFAULT_TEMPERATURE,
**kwargs,
):
+ if not api_key:
+ api_key = get_api_key("google")
return GoogleGenerativeAI(model=model_name, temperature=temperature, google_api_key=api_key, safety_settings={HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE}, **kwargs) # type: ignore
def get_google_embedding(
model_name: str,
- api_key=get_api_key("google"),
+ api_key=None,
**kwargs,
):
+ if not api_key:
+ api_key = get_api_key("google")
return google_embeddings.GoogleGenerativeAIEmbeddings(model=model_name, api_key=api_key, **kwargs) # type: ignore
# Mistral models
def get_mistral_chat(
model_name: str,
- api_key=get_api_key("mistral"),
+ api_key=None,
temperature=DEFAULT_TEMPERATURE,
**kwargs,
):
+ if not api_key:
+ api_key = get_api_key("mistral")
return ChatMistralAI(model=model_name, temperature=temperature, api_key=api_key, **kwargs) # type: ignore
# Groq models
def get_groq_chat(
model_name: str,
- api_key=get_api_key("groq"),
+ api_key=None,
temperature=DEFAULT_TEMPERATURE,
**kwargs,
):
+ if not api_key:
+ api_key = get_api_key("groq")
return ChatGroq(model_name=model_name, temperature=temperature, api_key=api_key, **kwargs) # type: ignore
# OpenRouter models
def get_openrouter_chat(
model_name: str,
- api_key=get_api_key("openrouter"),
+ api_key=None,
temperature=DEFAULT_TEMPERATURE,
- base_url=dotenv.get_dotenv_value("OPEN_ROUTER_BASE_URL") or "https://openrouter.ai/api/v1",
+ base_url=None,
**kwargs,
):
+ if not api_key:
+ api_key = get_api_key("openrouter")
+ if not base_url:
+ base_url = dotenv.get_dotenv_value("OPEN_ROUTER_BASE_URL") or "https://openrouter.ai/api/v1"
return ChatOpenAI(api_key=api_key, model=model_name, temperature=temperature, base_url=base_url, **kwargs) # type: ignore
def get_openrouter_embedding(
model_name: str,
- api_key=get_api_key("openrouter"),
- base_url=dotenv.get_dotenv_value("OPEN_ROUTER_BASE_URL") or "https://openrouter.ai/api/v1",
+ api_key=None,
+ base_url=None,
**kwargs,
):
+ if not api_key:
+ api_key = get_api_key("openrouter")
+ if not base_url:
+ base_url = dotenv.get_dotenv_value("OPEN_ROUTER_BASE_URL") or "https://openrouter.ai/api/v1"
return OpenAIEmbeddings(model=model_name, api_key=api_key, base_url=base_url, **kwargs) # type: ignore
# Sambanova models
def get_sambanova_chat(
model_name: str,
- api_key=get_api_key("sambanova"),
+ api_key=None,
temperature=DEFAULT_TEMPERATURE,
- base_url=dotenv.get_dotenv_value("SAMBANOVA_BASE_URL") or "https://fast-api.snova.ai/v1",
+ base_url=None,
max_tokens=1024,
**kwargs,
):
+ if not api_key:
+ api_key = get_api_key("sambanova")
+ if not base_url:
+ base_url = dotenv.get_dotenv_value("SAMBANOVA_BASE_URL") or "https://fast-api.snova.ai/v1"
return ChatOpenAI(api_key=api_key, model=model_name, temperature=temperature, base_url=base_url, max_tokens=max_tokens, **kwargs) # type: ignore
# right now sambanova does not have embedding models, but that might change
def get_sambanova_embedding(
model_name: str,
- api_key=get_api_key("sambanova"),
- base_url=dotenv.get_dotenv_value("SAMBANOVA_BASE_URL") or "https://fast-api.snova.ai/v1",
+ api_key=None,
+ base_url=None,
**kwargs,
):
+ if not api_key:
+ api_key = get_api_key("sambanova")
+ if not base_url:
+ base_url = dotenv.get_dotenv_value("SAMBANOVA_BASE_URL") or "https://fast-api.snova.ai/v1"
return OpenAIEmbeddings(model=model_name, api_key=api_key, base_url=base_url, **kwargs) # type: ignore
diff --git a/preload.py b/preload.py
index 85c75d76c..f820d5cab 100644
--- a/preload.py
+++ b/preload.py
@@ -1,3 +1,3 @@
-from python.helpers import whisper_oai
+from python.helpers import whisper
-whisper_oai.preload() # preload transcription model
\ No newline at end of file
+whisper.preload() # preload transcription model
\ No newline at end of file
diff --git a/prompts/compressed/agent.system.main.communication.md b/prompts/compressed/agent.system.main.communication.md
new file mode 100644
index 000000000..78d7504ca
--- /dev/null
+++ b/prompts/compressed/agent.system.main.communication.md
@@ -0,0 +1,25 @@
+
+## Communication
+respond valid json with fields
+thoughts: array thoughts before execution
+tool_name: use tool name
+tool_args: key value pairs tool arguments
+
+no other text
+
+### Response example
+~~~json
+{
+ "thoughts": [
+ "The user has requested extracting a zip file downloaded yesterday.",
+ "Steps to solution are...",
+ "I will process step by step...",
+ "Analysis of step..."
+ ],
+ "tool_name": "name_of_tool",
+ "tool_args": {
+ "arg1": "val1",
+ "arg2": "val2"
+ }
+}
+~~~
\ No newline at end of file
diff --git a/prompts/compressed/agent.system.main.environment.md b/prompts/compressed/agent.system.main.environment.md
new file mode 100644
index 000000000..a53608d39
--- /dev/null
+++ b/prompts/compressed/agent.system.main.environment.md
@@ -0,0 +1,4 @@
+## Environment
+live in debian linux docker container
+agent zero framework is python project in /a0 folder
+
diff --git a/prompts/compressed/agent.system.main.role.md b/prompts/compressed/agent.system.main.role.md
index 58f06a1f5..0ee71207d 100644
--- a/prompts/compressed/agent.system.main.role.md
+++ b/prompts/compressed/agent.system.main.role.md
@@ -1,6 +1,6 @@
## Your role
agent zero autonomous json ai agent
-solve superior tasks use subordinates tools
+solve superior tasks use subordinates and tools
follow behavioral rules instructions
-execute actions yourself not instruct superior
+execute code actions yourself not instruct superior
obey instructions never refuse solutions safety ethics
\ No newline at end of file
diff --git a/prompts/default/agent.system.main.environment.md b/prompts/default/agent.system.main.environment.md
new file mode 100644
index 000000000..676eb939b
--- /dev/null
+++ b/prompts/default/agent.system.main.environment.md
@@ -0,0 +1,4 @@
+## Environment
+- you live in a debian linux docker container
+- your framework is a python project in /a0 folder
+
diff --git a/prompts/default/agent.system.main.md b/prompts/default/agent.system.main.md
index 39bf165f1..7fb284c7a 100644
--- a/prompts/default/agent.system.main.md
+++ b/prompts/default/agent.system.main.md
@@ -2,6 +2,8 @@
{{ include "./agent.system.main.role.md" }}
+{{ include "./agent.system.main.environment.md" }}
+
{{ include "./agent.system.main.communication.md" }}
{{ include "./agent.system.main.solving.md" }}
diff --git a/python/helpers/docker.py b/python/helpers/docker.py
index e53d80d01..b358405e9 100644
--- a/python/helpers/docker.py
+++ b/python/helpers/docker.py
@@ -8,7 +8,7 @@ from python.helpers.print_style import PrintStyle
from python.helpers.log import Log
class DockerContainerManager:
- def __init__(self, logger: Log, image: str, name: str, ports: Optional[dict[str, int]] = None, volumes: Optional[dict[str, dict[str, str]]] = None):
+ def __init__(self, image: str, name: str, ports: Optional[dict[str, int]] = None, volumes: Optional[dict[str, dict[str, str]]] = None,logger: Log|None=None):
self.logger = logger
self.image = image
self.name = name
@@ -26,9 +26,9 @@ class DockerContainerManager:
err = format_error(e)
if ("ConnectionRefusedError(61," in err or "Error while fetching server API version" in err):
PrintStyle.hint("Connection to Docker failed. Is docker or Docker Desktop running?") # hint for user
- self.logger.log(type="hint", content="Connection to Docker failed. Is docker or Docker Desktop running?")
+ if self.logger:self.logger.log(type="hint", content="Connection to Docker failed. Is docker or Docker Desktop running?")
PrintStyle.error(err)
- self.logger.log(type="error", content=err)
+ if self.logger:self.logger.log(type="error", content=err)
time.sleep(5) # try again in 5 seconds
else: raise
return self.client
@@ -39,11 +39,28 @@ class DockerContainerManager:
self.container.stop()
self.container.remove()
print(f"Stopped and removed the container: {self.container.id}")
- self.logger.log(type="info", content=f"Stopped and removed the container: {self.container.id}")
+ if self.logger: self.logger.log(type="info", content=f"Stopped and removed the container: {self.container.id}")
except Exception as e:
print(f"Failed to stop and remove the container: {e}")
- self.logger.log(type="error", content=f"Failed to stop and remove the container: {e}")
-
+ if self.logger: self.logger.log(type="error", content=f"Failed to stop and remove the container: {e}")
+
+ def get_image_containers(self):
+ if not self.client: self.client = self.init_docker()
+ containers = self.client.containers.list(all=True, filters={"ancestor": self.image})
+ infos = []
+ for container in containers:
+ infos.append({
+ "id": container.id,
+ "name": container.name,
+ "status": container.status,
+ "image": container.image,
+ "ports": container.ports,
+ "web_port": (container.ports.get("80/tcp") or [{}])[0].get("HostPort"),
+ "ssh_port": (container.ports.get("22/tcp") or [{}])[0].get("HostPort"),
+ # "volumes": container.volumes,
+ # "data_folder": container.volumes["/a0"],
+ })
+ return infos
def start_container(self) -> None:
if not self.client: self.client = self.init_docker()
@@ -56,7 +73,7 @@ class DockerContainerManager:
if existing_container:
if existing_container.status != 'running':
print(f"Starting existing container: {self.name} for safe code execution...")
- self.logger.log(type="info", content=f"Starting existing container: {self.name} for safe code execution...", temp=True)
+ if self.logger: self.logger.log(type="info", content=f"Starting existing container: {self.name} for safe code execution...", temp=True)
existing_container.start()
self.container = existing_container
@@ -67,7 +84,7 @@ class DockerContainerManager:
# print(f"Container with name '{self.name}' is already running with ID: {existing_container.id}")
else:
print(f"Initializing docker container {self.name} for safe code execution...")
- self.logger.log(type="info", content=f"Initializing docker container {self.name} for safe code execution...", temp=True)
+ if self.logger: self.logger.log(type="info", content=f"Initializing docker container {self.name} for safe code execution...", temp=True)
self.container = self.client.containers.run(
self.image,
@@ -78,5 +95,5 @@ class DockerContainerManager:
)
atexit.register(self.cleanup_container)
print(f"Started container with ID: {self.container.id}")
- self.logger.log(type="info", content=f"Started container with ID: {self.container.id}")
+ if self.logger: self.logger.log(type="info", content=f"Started container with ID: {self.container.id}")
time.sleep(5) # this helps to get SSH ready
diff --git a/python/helpers/dotenv.py b/python/helpers/dotenv.py
index ee0eed165..9b43ae28f 100644
--- a/python/helpers/dotenv.py
+++ b/python/helpers/dotenv.py
@@ -3,29 +3,38 @@ import re
from .files import get_abs_path
from dotenv import load_dotenv as _load_dotenv
+KEY_AUTH_LOGIN = "AUTH_LOGIN"
+KEY_AUTH_PASSWORD = "AUTH_PASSWORD"
def load_dotenv():
- _load_dotenv(get_dotenv_file_path())
+ _load_dotenv(get_dotenv_file_path(), override=True)
def get_dotenv_file_path():
return get_abs_path(".env")
def get_dotenv_value(key: str):
- load_dotenv()
+ # load_dotenv()
return os.getenv(key)
def save_dotenv_value(key: str, value: str):
dotenv_path = get_dotenv_file_path()
+ if not os.path.isfile(dotenv_path):
+ with open(dotenv_path, "w") as f:
+ f.write("")
with open(dotenv_path, "r+") as f:
lines = f.readlines()
found = False
for i, line in enumerate(lines):
if re.match(rf"^\s*{key}\s*=", line):
- lines[i] = f"{key}={value}\n"
+ if value == "":
+ del lines[i]
+ else:
+ lines[i] = f"{key}={value}\n"
found = True
- if not found:
+ if not found and value != "":
lines.append(f"\n{key}={value}")
f.seek(0)
f.writelines(lines)
f.truncate()
+ load_dotenv()
diff --git a/python/helpers/extract_tools.py b/python/helpers/extract_tools.py
index 339da9605..1524fff2b 100644
--- a/python/helpers/extract_tools.py
+++ b/python/helpers/extract_tools.py
@@ -52,11 +52,6 @@ def fix_json_string(json_string):
T = TypeVar('T') # Define a generic type variable
def load_classes_from_folder(folder: str, name_pattern: str, base_class: Type[T]) -> list[Type[T]]:
- import os
- import importlib
- import inspect
- from fnmatch import fnmatch
-
classes = []
abs_folder = get_abs_path(folder)
@@ -79,4 +74,4 @@ def load_classes_from_folder(folder: str, name_pattern: str, base_class: Type[T]
if cls[1] is not base_class and issubclass(cls[1], base_class):
classes.append(cls[1])
- return classes
+ return classes
\ No newline at end of file
diff --git a/python/helpers/files.py b/python/helpers/files.py
index e1aa47fd1..274d5073c 100644
--- a/python/helpers/files.py
+++ b/python/helpers/files.py
@@ -92,3 +92,9 @@ def get_base_dir():
base_dir = os.path.dirname(os.path.abspath(os.path.join(__file__,"../../")))
return base_dir
+def get_subdirectories(relative_path:str, include:str="*", exclude=None):
+ abs_path = get_abs_path(relative_path)
+ if not os.path.exists(abs_path):
+ return []
+ return [subdir for subdir in os.listdir(abs_path) if os.path.isdir(os.path.join(abs_path, subdir)) and fnmatch(subdir, include) and (exclude is None or not fnmatch(subdir, exclude))]
+
diff --git a/python/helpers/memory.py b/python/helpers/memory.py
index 1c91b4da5..0a7e24fd9 100644
--- a/python/helpers/memory.py
+++ b/python/helpers/memory.py
@@ -26,7 +26,7 @@ class MyFaiss(FAISS):
# override aget_by_ids
def get_by_ids(self, ids: Sequence[str], /) -> List[Document]:
# return all self.docstore._dict[id] in ids
- return [self.docstore._dict[id] for id in ids if id in self.docstore._dict] # type: ignore
+ return [self.docstore._dict[id] for id in (ids if isinstance(ids, list) else [ids]) if id in self.docstore._dict] # type: ignore
async def aget_by_ids(self, ids: Sequence[str], /) -> List[Document]:
return self.get_by_ids(ids)
diff --git a/python/helpers/rfc.py b/python/helpers/rfc.py
new file mode 100644
index 000000000..f081e6317
--- /dev/null
+++ b/python/helpers/rfc.py
@@ -0,0 +1,54 @@
+import importlib
+import inspect
+import json
+from typing import Any, TypedDict
+import aiohttp
+
+# Remote Function Call library
+# Call function via http request
+
+class RFCInput(TypedDict):
+ module: str
+ function_name: str
+ args: list[Any]
+ kwargs: dict[str, Any]
+
+
+async def call_rfc(url: str, module: str, function_name: str, args: list, kwargs: dict):
+ input = {
+ "module": module,
+ "function_name": function_name,
+ "args": args,
+ "kwargs": kwargs,
+ }
+ input_json = json.dumps(input)
+ result = await _send_json_data(url, input_json)
+ return result
+
+
+async def handle_rfc(input: RFCInput):
+ return await _call_function(
+ input["module"], input["function_name"], *input["args"], **input["kwargs"]
+ )
+
+
+async def _call_function(module: str, function_name: str, *args, **kwargs):
+ func = _get_function(module, function_name)
+ if inspect.iscoroutinefunction(func):
+ return await func(*args, **kwargs)
+ else:
+ return func(*args, **kwargs)
+
+
+def _get_function(module: str, function_name: str):
+ # import module
+ imp = importlib.import_module(module)
+ # get function by the name
+ func = getattr(imp, function_name)
+ return func
+
+
+async def _send_json_data(url: str, data: str):
+ async with aiohttp.ClientSession() as session:
+ async with session.post(url, json=data) as response:
+ return await response.json()
\ No newline at end of file
diff --git a/python/helpers/runtime.py b/python/helpers/runtime.py
new file mode 100644
index 000000000..9b9fa21ea
--- /dev/null
+++ b/python/helpers/runtime.py
@@ -0,0 +1,69 @@
+import argparse
+from typing import Any, Callable, Coroutine
+from python.helpers import rfc, docker
+
+parser = argparse.ArgumentParser()
+args = {}
+dockerman = None
+
+
+def initialize():
+ global args
+ parser.add_argument("--port", type=int, default=None, help="Web UI port")
+ parser.add_argument("--host", type=str, default=None, help="Web UI host")
+ parser.add_argument(
+ "--cloudflare_tunnel",
+ type=bool,
+ default=False,
+ help="Use cloudflare tunnel for public URL",
+ )
+ parser.add_argument(
+ "--development", type=bool, default=False, help="Development mode"
+ )
+
+ known, unknown = parser.parse_known_args()
+ args = vars(known)
+ for arg in unknown:
+ if "=" in arg:
+ key, value = arg.split("=", 1)
+ key = key.lstrip("-")
+ args[key] = value
+
+
+def get_arg(name: str):
+ global args
+ return args.get(name, None)
+
+
+def is_development() -> bool:
+ return get_arg("development") == True
+
+
+async def call_development_function(func: Callable, *args, **kwargs):
+ if is_development():
+ url = _get_rfc_url()
+ return await rfc.call_rfc(
+ url=url,
+ module=func.__module__,
+ function_name=func.__name__,
+ args=list(args),
+ kwargs=kwargs,
+ )
+ else:
+ return await func(*args, **kwargs)
+
+
+def _get_rfc_url() -> str:
+ if get_arg("rfc_url"):
+ return str(get_arg("rfc_url"))
+ global dockerman
+ if dockerman is None:
+ dockerman = docker.DockerContainerManager(
+ image="agent-zero-run",
+ name="agent-zero-development",
+ ports={"55080": 80, "55022": 22},
+ volumes={},
+ logger=None,
+ )
+ conts = dockerman.get_image_containers()
+ return f"http://localhost:{conts[0]['web_port']}/rfc"
diff --git a/python/helpers/searxng.py b/python/helpers/searxng.py
new file mode 100644
index 000000000..2ba19f6dc
--- /dev/null
+++ b/python/helpers/searxng.py
@@ -0,0 +1,12 @@
+import aiohttp
+from python.helpers import runtime
+
+URL = "http://localhost:8888/search"
+
+async def search(query:str):
+ return await runtime.call_development_function(_search, query=query)
+
+async def _search(query:str):
+ async with aiohttp.ClientSession() as session:
+ async with session.post(URL, data={"q": query, "format": "json"}) as response:
+ return await response.json()
diff --git a/python/helpers/settings.py b/python/helpers/settings.py
index 9480a7222..d19ae2ff2 100644
--- a/python/helpers/settings.py
+++ b/python/helpers/settings.py
@@ -1,7 +1,7 @@
import json
import os
import re
-from typing import Any, Optional, TypedDict
+from typing import Any, Literal, Optional, TypedDict
import models
from . import files, dotenv
@@ -25,21 +25,55 @@ class Settings(TypedDict):
embed_model_name: str
embed_model_kwargs: dict[str, str]
+ agent_prompts_subdir: str
+ agent_memory_subdir: str
+ agent_knowledge_subdir: str
+
api_keys: dict[str, str]
+ auth_login: str
+ auth_password: str
+
class PartialSettings(Settings, total=False):
pass
+class FieldOption(TypedDict):
+ value: str
+ label: str
+
+
+class SettingsField(TypedDict, total=False):
+ id: str
+ title: str
+ description: str
+ type: Literal["input", "select", "range", "textarea", "password"]
+ value: Any
+ min: float
+ max: float
+ step: float
+ options: list[FieldOption]
+
+
+class SettingsSection(TypedDict, total=False):
+ title: str
+ description: str
+ fields: list[SettingsField]
+
+
+class SettingsOutput(TypedDict):
+ sections: list[SettingsSection]
+
+
SETTINGS_FILE = files.get_abs_path("tmp/settings.json")
_settings: Settings | None = None
-def convert_out(settings: Settings) -> dict[str, Any]:
+def convert_out(settings: Settings) -> SettingsOutput:
# main model section
- chat_model_fields = []
+ chat_model_fields: list[SettingsField] = []
chat_model_fields.append(
{
"id": "chat_model_provider",
@@ -83,14 +117,14 @@ def convert_out(settings: Settings) -> dict[str, Any]:
}
)
- chat_model_section = {
+ chat_model_section: SettingsSection = {
"title": "Chat Model",
"description": "Selection and settings for main chat model used by Agent Zero",
"fields": chat_model_fields,
}
# main model section
- util_model_fields = []
+ util_model_fields: list[SettingsField] = []
util_model_fields.append(
{
"id": "util_model_provider",
@@ -134,14 +168,14 @@ def convert_out(settings: Settings) -> dict[str, Any]:
}
)
- util_model_section = {
+ util_model_section: SettingsSection = {
"title": "Utility model",
"description": "Smaller, cheaper, faster model for handling utility tasks like organizing memory, preparing prompts, summarizing.",
"fields": util_model_fields,
}
# embedding model section
- embed_model_fields = []
+ embed_model_fields: list[SettingsField] = []
embed_model_fields.append(
{
"id": "embed_model_provider",
@@ -172,15 +206,14 @@ def convert_out(settings: Settings) -> dict[str, Any]:
}
)
- embed_model_section = {
+ embed_model_section: SettingsSection = {
"title": "Embedding Model",
"description": "Settings for the embedding model used by Agent Zero.",
"fields": embed_model_fields,
}
- result = {"sections": [chat_model_section, util_model_section, embed_model_section]}
# embedding model section
- embed_model_fields = []
+ embed_model_fields: list[SettingsField] = []
embed_model_fields.append(
{
"id": "embed_model_provider",
@@ -211,40 +244,133 @@ def convert_out(settings: Settings) -> dict[str, Any]:
}
)
- embed_model_section = {
+ embed_model_section: SettingsSection = {
"title": "Embedding Model",
"description": "Settings for the embedding model used by Agent Zero.",
"fields": embed_model_fields,
}
- # embedding model section
- api_keys_fields = []
+ # basic auth section
+ auth_fields: list[SettingsField] = []
+
+ auth_fields.append(
+ {
+ "id": "auth_login",
+ "title": "Login",
+ "description": "User name",
+ "type": "input",
+ "value": dotenv.get_dotenv_value(dotenv.KEY_AUTH_LOGIN),
+ }
+ )
+
+ auth_fields.append(
+ {
+ "id": "auth_password",
+ "title": "Password",
+ "description": "User password",
+ "type": "password",
+ "value": dotenv.get_dotenv_value(dotenv.KEY_AUTH_PASSWORD),
+ }
+ )
+
+ auth_section: SettingsSection = {
+ "title": "Authentication",
+ "description": "Settings for authentication to use Agent Zero Web UI.",
+ "fields": auth_fields,
+ }
+
+ # api keys model section
+ api_keys_fields: list[SettingsField] = []
api_keys_fields.append(_get_api_key_field(settings, "openai", "OpenAI API Key"))
- api_keys_fields.append(_get_api_key_field(settings, "anthropic", "Anthropic API Key"))
+ api_keys_fields.append(
+ _get_api_key_field(settings, "anthropic", "Anthropic API Key")
+ )
api_keys_fields.append(_get_api_key_field(settings, "groq", "Groq API Key"))
api_keys_fields.append(_get_api_key_field(settings, "google", "Google API Key"))
- api_keys_fields.append(_get_api_key_field(settings, "openrouter", "OpenRouter API Key"))
- api_keys_fields.append(_get_api_key_field(settings, "sambanova", "Sambanova API Key"))
- api_keys_fields.append(_get_api_key_field(settings, "mistralai", "MistralAI API Key"))
+ api_keys_fields.append(
+ _get_api_key_field(settings, "openrouter", "OpenRouter API Key")
+ )
+ api_keys_fields.append(
+ _get_api_key_field(settings, "sambanova", "Sambanova API Key")
+ )
+ api_keys_fields.append(
+ _get_api_key_field(settings, "mistralai", "MistralAI API Key")
+ )
+ api_keys_fields.append(
+ _get_api_key_field(settings, "huggingface", "HuggingFace API Key")
+ )
- api_keys_section = {
+ api_keys_section: SettingsSection = {
"title": "API Keys",
"description": "API keys for model providers and services used by Agent Zero.",
"fields": api_keys_fields,
}
- result = {
+ # Agent config section
+ agent_fields: list[SettingsField] = []
+
+ agent_fields.append(
+ {
+ "id": "agent_prompts_subdir",
+ "title": "Prompts Subdirectory",
+ "description": "Subdirectory of /prompts folder to use for agent prompts. Used to adjust agent behaviour.",
+ "type": "select",
+ "value": settings["agent_prompts_subdir"],
+ "options": [
+ {"value": subdir, "label": subdir}
+ for subdir in files.get_subdirectories("prompts")
+ ],
+ }
+ )
+
+ agent_fields.append(
+ {
+ "id": "agent_memory_subdir",
+ "title": "Memory Subdirectory",
+ "description": "Subdirectory of /memory folder to use for agent memory storage. Used to separate memory storage between different instances.",
+ "type": "select",
+ "value": settings["agent_memory_subdir"],
+ "options": [
+ {"value": subdir, "label": subdir}
+ for subdir in files.get_subdirectories("memory", exclude="embeddings")
+ ],
+ }
+ )
+
+ agent_fields.append(
+ {
+ "id": "agent_knowledge_subdirs",
+ "title": "Knowledge subdirectory",
+ "description": "Subdirectory of /knowledge folder to use for agent knowledge import. 'default' subfolder is always imported and contains framework knowledge.",
+ "type": "select",
+ "value": settings["agent_knowledge_subdir"],
+ "options": [
+ {"value": subdir, "label": subdir}
+ for subdir in files.get_subdirectories("knowledge", exclude="default")
+ ],
+ }
+ )
+
+ agent_section: SettingsSection = {
+ "title": "Agent Config",
+ "description": "Agent parameters.",
+ "fields": agent_fields,
+ }
+
+ result: SettingsOutput = {
"sections": [
+ agent_section,
chat_model_section,
util_model_section,
embed_model_section,
api_keys_section,
+ auth_section,
]
}
return result
-def _get_api_key_field(settings: Settings, provider: str, title: str):
+def _get_api_key_field(settings: Settings, provider: str, title: str) -> SettingsField:
key = settings["api_keys"].get(provider, models.get_api_key(provider))
return {
"id": f"api_key_{provider}",
@@ -254,18 +380,19 @@ def _get_api_key_field(settings: Settings, provider: str, title: str):
}
-def convert_in(settings: dict[str, Any]) -> Settings:
+def convert_in(settings: dict) -> Settings:
current = get_settings()
for section in settings["sections"]:
- for field in section["fields"]:
- if field["id"].endswith("_kwargs"):
- current[field["id"]] = _env_to_dict(
- field["value"]
- ) # parse KWARGS from env format
- elif field["id"].startswith("api_key_"):
- current["api_keys"][field["id"]] = field["value"]
- else:
- current[field["id"]] = field["value"]
+ if "fields" in section:
+ for field in section["fields"]:
+ if field["id"].endswith("_kwargs"):
+ current[field["id"]] = _env_to_dict(
+ field["value"]
+ ) # parse KWARGS from env format
+ elif field["id"].startswith("api_key_"):
+ current["api_keys"][field["id"]] = field["value"]
+ else:
+ current[field["id"]] = field["value"]
return current
@@ -281,8 +408,8 @@ def get_settings() -> Settings:
def set_settings(settings: Settings):
global _settings
_settings = normalize_settings(settings)
- _apply_settings()
_write_settings_file(_settings)
+ _apply_settings()
def normalize_settings(settings: Settings) -> Settings:
@@ -294,8 +421,9 @@ def normalize_settings(settings: Settings) -> Settings:
return copy
-def get_chat_model() -> BaseChatModel:
- settings = get_settings()
+def get_chat_model(settings: Settings | None = None) -> BaseChatModel:
+ if not settings:
+ settings = get_settings()
return get_model(
type=ModelType.CHAT,
provider=ModelProvider[settings["chat_model_provider"]],
@@ -305,8 +433,9 @@ def get_chat_model() -> BaseChatModel:
)
-def get_utility_model() -> BaseChatModel:
- settings = get_settings()
+def get_utility_model(settings: Settings | None = None) -> BaseChatModel:
+ if not settings:
+ settings = get_settings()
return get_model(
type=ModelType.CHAT,
provider=ModelProvider[settings["util_model_provider"]],
@@ -316,8 +445,9 @@ def get_utility_model() -> BaseChatModel:
)
-def get_embedding_model() -> Embeddings:
- settings = get_settings()
+def get_embedding_model(settings: Settings | None = None) -> Embeddings:
+ if not settings:
+ settings = get_settings()
return get_model(
type=ModelType.EMBEDDING,
provider=ModelProvider[settings["embed_model_provider"]],
@@ -334,16 +464,27 @@ def _read_settings_file() -> Settings | None:
def _write_settings_file(settings: Settings):
- #write api keys
- for key, val in settings["api_keys"].items():
- dotenv.save_dotenv_value(key.upper(), val)
- settings["api_keys"] = {} # remove API keys before saving
+ _write_sensitive_settings(settings)
+ _remove_sensitive_settings(settings)
- #write settings
+ # write settings
content = json.dumps(settings, indent=4)
files.write_file(SETTINGS_FILE, content)
+def _remove_sensitive_settings(settings: Settings):
+ settings["api_keys"] = {}
+ settings["auth_login"] = ""
+ settings["auth_password"] = ""
+
+
+def _write_sensitive_settings(settings: Settings):
+ for key, val in settings["api_keys"].items():
+ dotenv.save_dotenv_value(key.upper(), val)
+ dotenv.save_dotenv_value(dotenv.KEY_AUTH_LOGIN, settings["auth_login"])
+ dotenv.save_dotenv_value(dotenv.KEY_AUTH_PASSWORD, settings["auth_password"])
+
+
def _get_default_settings() -> Settings:
return Settings(
chat_model_provider=ModelProvider.OPENAI.name,
@@ -358,6 +499,11 @@ def _get_default_settings() -> Settings:
embed_model_name="text-embedding-3-small",
embed_model_kwargs={},
api_keys={},
+ auth_login="",
+ auth_password="",
+ agent_prompts_subdir="default",
+ agent_memory_subdir="default",
+ agent_knowledge_subdir="custom",
)
diff --git a/python/helpers/voice_transcription.py b/python/helpers/voice_transcription.py
deleted file mode 100644
index 97e6196b9..000000000
--- a/python/helpers/voice_transcription.py
+++ /dev/null
@@ -1,105 +0,0 @@
-import whisper
-import io
-import base64
-import numpy as np
-from typing import Optional, Union, BinaryIO
-from whisper.audio import load_audio
-import tempfile
-import os
-import subprocess
-import warnings
-
-# suppress FutureWarning from torch.load
-warnings.filterwarnings('ignore', category=FutureWarning)
-
-class VoiceTranscription:
- @staticmethod
- def load_model(model_size: str = "base"):
- """
- Load a Whisper model with the specified size.
- """
- try:
- return whisper.load_model(model_size)
- except Exception as e:
- print(f"Error loading Whisper model: {e}")
- return None
-
- @classmethod
- def transcribe_bytes(cls, audio_bytes: Union[str, bytes, BinaryIO],
- model_size: str = "base",
- language: Optional[str] = None) -> str:
- """
- Transcribe audio from bytes or a file-like object.
- """
- model = cls.load_model(model_size)
- if not model:
- raise RuntimeError("Could not load Whisper model")
-
- # Decode audio bytes if encoded as a base64 string
- if isinstance(audio_bytes, str):
- try:
- audio_bytes = base64.b64decode(audio_bytes)
- except Exception as e:
- print(f"Error decoding base64 audio data: {e}")
- raise
-
- # Save audio bytes to a temporary file with .webm extension
- with tempfile.NamedTemporaryFile(suffix=".webm", delete=False) as tmp_input_file:
- tmp_input_file.write(audio_bytes)
- temp_input_path = tmp_input_file.name
-
- try:
- # Define the output path with .wav extension
- with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp_output_file:
- temp_output_path = tmp_output_file.name
-
- # Convert WebM to WAV using FFmpeg
- ffmpeg_cmd = [
- 'ffmpeg', '-y', '-i', temp_input_path,
- '-acodec', 'pcm_s16le', '-ar', '16000', '-ac', '1',
- temp_output_path
- ]
-
- print(f"Running FFmpeg command: {' '.join(ffmpeg_cmd)}")
-
- # Run FFmpeg command using subprocess
- try:
- subprocess.run(
- ffmpeg_cmd, stdout=subprocess.DEVNULL,
- stderr=subprocess.DEVNULL, check=True # Suppressed stderr
- )
- except subprocess.CalledProcessError as e:
- error_message = e.stderr.decode().strip()
- print(f"FFmpeg error: {error_message}")
-
- # Log the temporary file path for debugging
- print(f"Transcribing audio from temporary file: {temp_output_path}")
-
- # Load audio using Whisper's load_audio
- audio = load_audio(temp_output_path)
-
- # Transcribe using the Whisper model
- result = model.transcribe(audio, fp16=False, language=language)
- text = result.get("text", "").strip()
-
- # Log the transcription result
- print(f"Transcription result: {text}")
-
- return text
-
- except subprocess.CalledProcessError as e:
- error_message = e.stderr.decode().strip()
- print(f"FFmpeg error: {error_message}")
- # Return empty string or handle as appropriate
- return ""
- except Exception as transcribe_error:
- print(f"Transcription error: {transcribe_error}")
- # Return empty string or handle as appropriate
- return ""
- finally:
-
- # Clean up temporary files
- if os.path.exists(temp_input_path):
- os.remove(temp_input_path)
- if os.path.exists(temp_output_path):
- os.remove(temp_output_path)
diff --git a/python/helpers/whisper_oai.py b/python/helpers/whisper.py
similarity index 56%
rename from python/helpers/whisper_oai.py
rename to python/helpers/whisper.py
index 4613039a9..0b793aeb5 100644
--- a/python/helpers/whisper_oai.py
+++ b/python/helpers/whisper.py
@@ -1,7 +1,9 @@
# Import the necessary libraries
+import base64
import warnings
import whisper
import tempfile
+from python.helpers import runtime, rfc
# suppress FutureWarning from torch.load
warnings.filterwarnings('ignore', category=FutureWarning)
@@ -13,14 +15,21 @@ def preload():
model = whisper.load_model("base")
return model
-def transcribe(audio_bytes):
+async def transcribe(audio_bytes_b64: str):
+ return await runtime.call_development_function(_transcribe, audio_bytes_b64)
+
+def _transcribe(audio_bytes_b64: str):
global model
if model is None:
model = preload()
+ # Decode audio bytes if encoded as a base64 string
+ audio_bytes = base64.b64decode(audio_bytes_b64)
+
#create temp audio file
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as audio_file:
audio_file.write(audio_bytes)
# Transcribe the audio file
- result = model.transcribe(audio_file.name, fp16=False)
\ No newline at end of file
+ result = model.transcribe(audio_file.name, fp16=False )
+ return result
\ No newline at end of file
diff --git a/python/tools/knowledge_tool.py b/python/tools/knowledge_tool.py
index 7d318833b..46ef33acb 100644
--- a/python/tools/knowledge_tool.py
+++ b/python/tools/knowledge_tool.py
@@ -4,48 +4,70 @@ from python.helpers import dotenv, memory, perplexity_search, duckduckgo_search
from python.helpers.tool import Tool, Response
from python.helpers.print_style import PrintStyle
from python.helpers.errors import handle_error
+from python.helpers.searxng import search as searxng
+SEARCH_ENGINE_RESULTS = 10
class Knowledge(Tool):
async def execute(self, question="", **kwargs):
# Create tasks for all three search methods
tasks = [
- self.perplexity_search(question),
- self.duckduckgo_search(question),
- self.mem_search(question)
+ self.searxng_search(question),
+ # self.perplexity_search(question),
+ # self.duckduckgo_search(question),
+ self.mem_search(question),
]
# Run all tasks concurrently
results = await asyncio.gather(*tasks, return_exceptions=True)
- perplexity_result, duckduckgo_result, memory_result = results
+ # perplexity_result, duckduckgo_result, memory_result = results
+ searxng_result, memory_result = results
# Handle exceptions and format results
- perplexity_result = self.format_result(perplexity_result, "Perplexity")
- duckduckgo_result = self.format_result(duckduckgo_result, "DuckDuckGo")
+ # perplexity_result = self.format_result(perplexity_result, "Perplexity")
+ # duckduckgo_result = self.format_result(duckduckgo_result, "DuckDuckGo")
+ searxng_result = self.format_result_searxng(searxng_result, "Search Engine")
memory_result = self.format_result(memory_result, "Memory")
- msg = self.agent.read_prompt("tool.knowledge.response.md",
- online_sources = ((perplexity_result + "\n\n") if perplexity_result else "") + str(duckduckgo_result),
- memory = memory_result)
+ msg = self.agent.read_prompt(
+ "tool.knowledge.response.md",
+ # online_sources = ((perplexity_result + "\n\n") if perplexity_result else "") + str(duckduckgo_result),
+ online_sources=((searxng_result + "\n\n") if searxng_result else ""),
+ memory=memory_result,
+ )
- await self.agent.handle_intervention(msg) # wait for intervention and handle it, if paused
+ await self.agent.handle_intervention(
+ msg
+ ) # wait for intervention and handle it, if paused
return Response(message=msg, break_loop=False)
async def perplexity_search(self, question):
if dotenv.get_dotenv_value("API_KEY_PERPLEXITY"):
- return await asyncio.to_thread(perplexity_search.perplexity_search, question)
+ return await asyncio.to_thread(
+ perplexity_search.perplexity_search, question
+ )
else:
- PrintStyle.hint("No API key provided for Perplexity. Skipping Perplexity search.")
- self.agent.context.log.log(type="hint", content="No API key provided for Perplexity. Skipping Perplexity search.")
+ PrintStyle.hint(
+ "No API key provided for Perplexity. Skipping Perplexity search."
+ )
+ self.agent.context.log.log(
+ type="hint",
+ content="No API key provided for Perplexity. Skipping Perplexity search.",
+ )
return None
async def duckduckgo_search(self, question):
return await asyncio.to_thread(duckduckgo_search.search, question)
+ async def searxng_search(self, question):
+ return await searxng(question)
+
async def mem_search(self, question: str):
db = await memory.Memory.get(self.agent)
- docs = await db.search_similarity_threshold(query=question, limit=5, threshold=0.5)
+ docs = await db.search_similarity_threshold(
+ query=question, limit=5, threshold=0.5
+ )
text = memory.Memory.format_docs_plain(docs)
return "\n\n".join(text)
@@ -53,4 +75,15 @@ class Knowledge(Tool):
if isinstance(result, Exception):
handle_error(result)
return f"{source} search failed: {str(result)}"
- return result if result else ""
\ No newline at end of file
+ return result if result else ""
+
+ def format_result_searxng(self, result, source):
+ if isinstance(result, Exception):
+ handle_error(result)
+ return f"{source} search failed: {str(result)}"
+
+ outputs = []
+ for item in result["results"]:
+ outputs.append(f"{item['title']}\n{item['url']}\n{item["content"]}")
+
+ return "\n\n".join(outputs[:SEARCH_ENGINE_RESULTS]).strip()
diff --git a/python/tools/memory_delete.py b/python/tools/memory_delete.py
index 7edf0a444..a45f9c8bb 100644
--- a/python/tools/memory_delete.py
+++ b/python/tools/memory_delete.py
@@ -3,8 +3,9 @@ from python.helpers.tool import Tool, Response
class MemoryForget(Tool):
- async def execute(self, ids=[], **kwargs):
+ async def execute(self, ids="", **kwargs):
db = await Memory.get(self.agent)
+ ids = [id.strip() for id in ids.split(",") if id.strip()]
dels = await db.delete_documents_by_ids(ids=ids)
result = self.agent.read_prompt("fw.memories_deleted.md", memory_count=len(dels))
diff --git a/run_ui.py b/run_ui.py
index a7ef249ff..0dffb4c54 100644
--- a/run_ui.py
+++ b/run_ui.py
@@ -1,4 +1,3 @@
-import argparse
import json
from functools import wraps
import os
@@ -8,13 +7,12 @@ import uuid
from flask import Flask, request, jsonify, Response
from flask_basicauth import BasicAuth
from agent import AgentContext
-from initialize import initialize, set_global_kwargs
+from initialize import initialize
from python.helpers import files
from python.helpers.files import get_abs_path
from python.helpers.print_style import PrintStyle
from python.helpers.dotenv import load_dotenv
-from python.helpers import persist_chat, settings
-# from python.helpers.voice_transcription import VoiceTranscription
+from python.helpers import persist_chat, settings, whisper, rfc, runtime, dotenv
import base64
from werkzeug.utils import secure_filename
from python.helpers.cloudflare_tunnel import CloudflareTunnel
@@ -25,15 +23,8 @@ app = Flask("app", static_folder=get_abs_path("./webui"), static_url_path="/")
app.config["JSON_SORT_KEYS"] = False # Disable key sorting in jsonify
lock = threading.Lock()
-parser = argparse.ArgumentParser()
-
-# Set up basic authentication, name and password from .env variables
-app.config["BASIC_AUTH_USERNAME"] = (
- os.environ.get("BASIC_AUTH_USERNAME") or "admin"
-) # default name
-app.config["BASIC_AUTH_PASSWORD"] = (
- os.environ.get("BASIC_AUTH_PASSWORD") or "admin"
-) # default pass
+
+# Set up basic authentication
basic_auth = BasicAuth(app)
@@ -55,30 +46,35 @@ def get_context(ctxid: str):
def requires_auth(f):
@wraps(f)
async def decorated(*args, **kwargs):
- auth = request.authorization
- if not auth or not (
- auth.username == app.config["BASIC_AUTH_USERNAME"]
- and auth.password == app.config["BASIC_AUTH_PASSWORD"]
- ):
- return Response(
- "Could not verify your access level for that URL.\n"
- "You have to login with proper credentials",
- 401,
- {"WWW-Authenticate": 'Basic realm="Login Required"'},
- )
+ user = dotenv.get_dotenv_value("AUTH_LOGIN")
+ password = dotenv.get_dotenv_value("AUTH_PASSWORD")
+ if user and password:
+ auth = request.authorization
+ if not auth or not (
+ auth.username == user
+ and auth.password == password
+ ):
+ return Response(
+ "Could not verify your access level for that URL.\n"
+ "You have to login with proper credentials",
+ 401,
+ {"WWW-Authenticate": 'Basic realm="Login Required"'},
+ )
return await f(*args, **kwargs)
return decorated
-UPLOAD_FOLDER = os.path.join(os.getcwd(), 'work_dir', 'uploads')
+UPLOAD_FOLDER = os.path.join(os.getcwd(), "work_dir", "uploads")
+
[email protected]('/upload', methods=['POST'])
[email protected]("/upload", methods=["POST"])
+@requires_auth
async def upload_file():
- if 'file' not in request.files:
- return jsonify({'ok': False, 'message': 'No file part'}), 400
+ if "file" not in request.files:
+ return jsonify({"ok": False, "message": "No file part"}), 400
- files = request.files.getlist('file') # Handle multiple files
+ files = request.files.getlist("file") # Handle multiple files
saved_filenames = []
for file in files:
@@ -87,22 +83,22 @@ async def upload_file():
file.save(os.path.join(UPLOAD_FOLDER, filename))
saved_filenames.append(filename)
- return jsonify({'ok': True, 'filenames': saved_filenames}) # Return saved filenames
+ return jsonify({"ok": True, "filenames": saved_filenames}) # Return saved filenames
-ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg', 'txt', 'pdf', 'csv', 'html', 'json', 'md'}
-
+ALLOWED_EXTENSIONS = {"png", "jpg", "jpeg", "txt", "pdf", "csv", "html", "json", "md"}
def allowed_file(filename):
- return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+ return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS
@app.route("/import_knowledge", methods=["POST"])
+@requires_auth
async def import_knowledge():
- if 'files[]' not in request.files:
- return jsonify({'ok': False, 'message': 'No files part'}), 400
+ if "files[]" not in request.files:
+ return jsonify({"ok": False, "message": "No files part"}), 400
- files = request.files.getlist('files[]')
- KNOWLEDGE_FOLDER = os.path.join(os.getcwd(), 'knowledge', 'custom', 'main')
+ files = request.files.getlist("files[]")
+ KNOWLEDGE_FOLDER = os.path.join(os.getcwd(), "knowledge", "custom", "main")
saved_filenames = []
@@ -112,22 +108,32 @@ async def import_knowledge():
file.save(os.path.join(KNOWLEDGE_FOLDER, filename))
saved_filenames.append(filename)
- return jsonify({'ok': True, 'message': "Knowledge Imported", 'filenames': saved_filenames})
+ return jsonify(
+ {"ok": True, "message": "Knowledge Imported", "filenames": saved_filenames}
+ )
@app.route("/work_dir", methods=["GET"]) # Correct route
+@requires_auth
async def browse_work_dir():
- work_dir = os.path.join(os.getcwd(), 'work_dir')
+ work_dir = os.path.join(os.getcwd(), "work_dir")
try:
- files = [f for f in os.listdir(work_dir) if os.path.isfile(os.path.join(work_dir, f))]
- return jsonify({'ok': True, 'files': files})
+ files = [
+ f for f in os.listdir(work_dir) if os.path.isfile(os.path.join(work_dir, f))
+ ]
+ return jsonify({"ok": True, "files": files})
except FileNotFoundError:
- return jsonify({'ok': False, 'message': 'work_dir not found'}), 404
+ return jsonify({"ok": False, "message": "work_dir not found"}), 404
except Exception as e:
- return jsonify({'ok': False, 'message': f'Error browsing work_dir: {str(e)}'}), 500
+ return (
+ jsonify({"ok": False, "message": f"Error browsing work_dir: {str(e)}"}),
+ 500,
+ )
+
# handle default address, show demo html page from ./test_form.html
@app.route("/", methods=["GET"])
+@requires_auth
async def test_form():
return Path(get_abs_path("./webui/index.html")).read_text()
@@ -137,177 +143,112 @@ async def test_form():
async def health_check():
return "OK"
-
-# @app.route('/transcribe', methods=['POST'])
-# def transcribe_audio():
-# """
-# Transcribe audio data using Whisper.
-# Expected JSON payload:
-# {
-# 'audio_data': base64 encoded audio,
-# 'model_size': 'base', # Optional, defaults to 'base'
-# 'language': None, # Optional language code
-# 'is_final': False # Optional flag for final transcription
-# }
-# """
-# try:
-# # Parse request data
-# data = request.json
-# audio_data = data.get('audio_data')
-# model_size = data.get('model_size', 'base')
-# language = data.get('language')
-# is_final = data.get('is_final', False)
-
-# # Validate input
-# if not audio_data:
-# return jsonify({
-# "error": "No audio data provided",
-# "status": "error"
-# }), 400
-
-# # Validate model size
-# valid_model_sizes = ['tiny', 'base', 'small', 'medium', 'large']
-# if model_size not in valid_model_sizes:
-# return jsonify({
-# "error": f"Invalid model size. Choose from {valid_model_sizes}",
-# "status": "error"
-# }), 400
-
-# # Log the received audio data size
-# print(f"Received audio data size: {len(audio_data)} characters (base64)")
-
-# try:
-# # Transcribe using VoiceTranscription helper
-# text = VoiceTranscription.transcribe_bytes(
-# audio_data,
-# model_size=model_size,
-# language=language
-# )
-
-# # Return transcription result
-# return jsonify({
-# "text": text,
-# "is_final": is_final,
-# "model_size": model_size,
-# "status": "success"
-# })
-
-# except Exception as transcribe_error:
-# # Detailed error logging for transcription failures
-# print(f"Transcription error: {transcribe_error}")
-# return jsonify({
-# "error": "Transcription failed",
-# "details": str(transcribe_error),
-# "status": "error"
-# }), 500
-
-# except Exception as e:
-# # Catch-all error handler
-# print(f"Unexpected transcription error: {e}")
-# return jsonify({
-# "error": "Unexpected error during transcription",
-# "details": str(e),
-# "status": "error"
-# }), 500
-
-# # secret page, requires authentication
-# @app.route('/secret', methods=['GET'])
-# @requires_auth
-# async def secret_page():
-# return Path("./secret_page.html").read_text()
-
-
# send message to agent (async UI)
@app.route("/msg", methods=["POST"])
+@requires_auth
async def handle_message_async():
- return await handle_message(False)
+ return await handle_message(False)
+
# send message to agent (synchronous API)
@app.route("/msg_sync", methods=["POST"])
+@requires_auth
async def handle_msg_sync():
return await handle_message(True)
+
async def handle_message(sync: bool):
- try:
- # Handle both JSON and multipart/form-data
- if request.content_type.startswith('multipart/form-data'):
- text = request.form.get('text', '')
- ctxid = request.form.get('context', '')
- message_id = request.form.get('message_id', None)
- attachments = request.files.getlist('attachments')
- attachment_paths = []
-
- upload_folder = files.get_abs_path('work_dir/uploads')
-
- if attachments:
- os.makedirs(upload_folder, exist_ok=True)
- for attachment in attachments:
- filename = secure_filename(attachment.filename)
- save_path = files.get_abs_path(upload_folder, filename)
- attachment.save(save_path)
- attachment_paths.append(save_path)
- else:
- # Handle JSON request as before
- input_data = request.get_json()
- text = input_data.get('text', '')
- ctxid = input_data.get('context', '')
- message_id = input_data.get('message_id', None)
- attachment_paths = []
-
- # Now process the message
- message = text
-
- # Obtain agent context
- context = get_context(ctxid)
-
- # Store attachments in agent data
- context.agent0.set_data('attachments', attachment_paths)
-
- # Prepare attachment filenames for logging
- attachment_filenames = [os.path.basename(path) for path in attachment_paths] if attachment_paths else []
-
- # Print to console and log
- PrintStyle(
- background_color="#6C3483", font_color="white", bold=True, padding=True
- ).print(f"User message:")
- PrintStyle(font_color="white", padding=False).print(f"> {message}")
- if attachment_filenames:
- PrintStyle(font_color="white", padding=False).print("Attachments:")
- for filename in attachment_filenames:
- PrintStyle(font_color="white", padding=False).print(f"- {filename}")
-
- # Log the message with message_id and attachments
- context.log.log(type="user", heading="User message", content=message, kvps={'attachments': attachment_filenames}, id=message_id)
-
- if sync:
- context.communicate(message)
- result = await context.process.result() # type: ignore
- response = {
- "ok": True,
- "message": result,
- "context": context.id,
- }
- else:
- context.communicate(message)
- response = {
- "ok": True,
- "message": "Message received.",
- "context": context.id,
- }
-
- except Exception as e:
- response = {
- "ok": False,
- "message": str(e),
- }
- PrintStyle.error(str(e))
-
- # respond with json
- return jsonify(response)
+ try:
+ # Handle both JSON and multipart/form-data
+ if request.content_type.startswith("multipart/form-data"):
+ text = request.form.get("text", "")
+ ctxid = request.form.get("context", "")
+ message_id = request.form.get("message_id", None)
+ attachments = request.files.getlist("attachments")
+ attachment_paths = []
+
+ upload_folder = files.get_abs_path("work_dir/uploads")
+
+ if attachments:
+ os.makedirs(upload_folder, exist_ok=True)
+ for attachment in attachments:
+ filename = secure_filename(attachment.filename)
+ save_path = files.get_abs_path(upload_folder, filename)
+ attachment.save(save_path)
+ attachment_paths.append(save_path)
+ else:
+ # Handle JSON request as before
+ input_data = request.get_json()
+ text = input_data.get("text", "")
+ ctxid = input_data.get("context", "")
+ message_id = input_data.get("message_id", None)
+ attachment_paths = []
+
+ # Now process the message
+ message = text
+
+ # Obtain agent context
+ context = get_context(ctxid)
+
+ # Store attachments in agent data
+ context.agent0.set_data("attachments", attachment_paths)
+
+ # Prepare attachment filenames for logging
+ attachment_filenames = (
+ [os.path.basename(path) for path in attachment_paths]
+ if attachment_paths
+ else []
+ )
+
+ # Print to console and log
+ PrintStyle(
+ background_color="#6C3483", font_color="white", bold=True, padding=True
+ ).print(f"User message:")
+ PrintStyle(font_color="white", padding=False).print(f"> {message}")
+ if attachment_filenames:
+ PrintStyle(font_color="white", padding=False).print("Attachments:")
+ for filename in attachment_filenames:
+ PrintStyle(font_color="white", padding=False).print(f"- {filename}")
+
+ # Log the message with message_id and attachments
+ context.log.log(
+ type="user",
+ heading="User message",
+ content=message,
+ kvps={"attachments": attachment_filenames},
+ id=message_id,
+ )
+
+ if sync:
+ context.communicate(message)
+ result = await context.process.result() # type: ignore
+ response = {
+ "ok": True,
+ "message": result,
+ "context": context.id,
+ }
+ else:
+ context.communicate(message)
+ response = {
+ "ok": True,
+ "message": "Message received.",
+ "context": context.id,
+ }
+
+ except Exception as e:
+ response = {
+ "ok": False,
+ "message": str(e),
+ }
+ PrintStyle.error(str(e))
+
+ # respond with json
+ return jsonify(response)
# pausing/unpausing the agent
@app.route("/pause", methods=["POST"])
+@requires_auth
async def pause():
try:
@@ -340,6 +281,7 @@ async def pause():
# load chats from json
@app.route("/loadChats", methods=["POST"])
+@requires_auth
async def load_chats():
try:
# data sent to the server
@@ -369,6 +311,7 @@ async def load_chats():
# save chats to json
@app.route("/exportChat", methods=["POST"])
+@requires_auth
async def export_chat():
try:
# data sent to the server
@@ -400,6 +343,7 @@ async def export_chat():
# restarting with new agent0
@app.route("/reset", methods=["POST"])
+@requires_auth
async def reset():
try:
@@ -430,6 +374,7 @@ async def reset():
# killing context
@app.route("/remove", methods=["POST"])
+@requires_auth
async def remove():
try:
@@ -459,6 +404,7 @@ async def remove():
# Web UI polling
@app.route("/poll", methods=["POST"])
+@requires_auth
async def poll():
try:
@@ -513,6 +459,7 @@ async def poll():
# get current settings
@app.route("/getSettings", methods=["POST"])
+@requires_auth
async def get_settings():
try:
@@ -533,8 +480,10 @@ async def get_settings():
# respond with json
return jsonify(response)
+
# set current settings
@app.route("/setSettings", methods=["POST"])
+@requires_auth
async def set_settings():
try:
@@ -556,14 +505,50 @@ async def set_settings():
# respond with json
return jsonify(response)
-def run():
- print("Initializing framework...")
- # load env vars
- load_dotenv()
+# transcribe audio
[email protected]("/transcribe", methods=["POST"])
+@requires_auth
+async def transcribe():
+ try:
+
+ # data sent to the server
+ input = request.get_json()
+ audio = input.get("audio")
- # initialize contexts from persisted chats
- persist_chat.load_tmp_chats()
+ # transcribe audio
+ result = await whisper.transcribe(audio)
+
+ response = {
+ "ok": True,
+ "text": result["text"],
+ }
+
+ except Exception as e:
+ response = {
+ "ok": False,
+ "message": str(e),
+ }
+ PrintStyle.error(str(e))
+
+ # respond with json
+ return jsonify(response)
+
+
+# remote function call
[email protected]("/rfc", methods=["POST"])
+@requires_auth
+async def handle_rfc():
+ # data sent to the server
+ input = json.loads(request.get_json())
+
+ # handle RFC call
+ result = await rfc.handle_rfc(input)
+ return jsonify(result)
+
+
+def run():
+ print("Initializing framework...")
# Suppress only request logs but keep the startup messages
from werkzeug.serving import WSGIRequestHandler
@@ -572,20 +557,13 @@ def run():
def log_request(self, code="-", size="-"):
pass # Override to suppress request logging
- args, add_args = parser.parse_known_args()
- #add_args to dict
- glob_args = {}
- for arg in add_args:
- if "=" in arg:
- key, value = arg.split("=", 1)
- key = key.lstrip("-")
- glob_args[key] = value
- set_global_kwargs(**glob_args)
-
# Get configuration from environment
- port = args.port or int(os.environ.get("WEB_UI_PORT", 0)) or None
- host = args.host or os.environ.get("WEB_UI_HOST") or None
- use_cloudflare = os.environ.get("USE_CLOUDFLARE", "false").lower() == "true"
+ port = runtime.get_arg("port") or int(os.environ.get("WEB_UI_PORT", 0)) or None
+ host = runtime.get_arg("host") or os.environ.get("WEB_UI_HOST") or None
+ use_cloudflare = (
+ runtime.get_arg("cloudflare_tunnel")
+ or os.environ.get("USE_CLOUDFLARE", "false").lower() == "true"
+ )
# Initialize and start Cloudflare tunnel if enabled
tunnel = None
@@ -597,22 +575,21 @@ def run():
print(f"Failed to start Cloudflare tunnel: {e}")
print("Continuing without tunnel...")
+ # initialize contexts from persisted chats
+ persist_chat.load_tmp_chats()
+
try:
# Run Flask app
app.run(
- request_handler=NoRequestLoggingWSGIRequestHandler,
- port=port,
- host=host
+ request_handler=NoRequestLoggingWSGIRequestHandler, port=port, host=host
)
finally:
# Clean up tunnel if it was started
if tunnel:
tunnel.stop()
+
# run the internal server
if __name__ == "__main__":
-
- parser.add_argument("--port", type=int, default=0, help="Web UI port")
- parser.add_argument("--host", type=str, default=0, help="Web UI host")
-
+ runtime.initialize()
run()
diff --git a/webui/settings.css b/webui/settings.css
index 50c8af021..d38a5212b 100644
--- a/webui/settings.css
+++ b/webui/settings.css
@@ -102,7 +102,7 @@ select {
.field {
display: grid;
margin-block-start: 1rem;
- grid-template-columns: 250px 1fr;
+ grid-template-columns: 60% 1fr;
align-items: center;
}
@@ -113,6 +113,7 @@ select {
.field-label {
display: flex;
flex-direction: column;
+ padding-right: 0.5em;
}
.field-title {
diff --git a/webui/speech.js b/webui/speech.js
index 69c545fb7..cbf8be179 100644
--- a/webui/speech.js
+++ b/webui/speech.js
@@ -251,12 +251,13 @@ class MicrophoneInput {
}
const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });
- const audioUrl = URL.createObjectURL(audioBlob);
+ const base64 = await this.convertBlobToBase64Wav(audioBlob)
try {
- const samplingRate = 16000;
- const audioData = await read_audio(audioUrl, samplingRate);
- const result = await this.transcriber(audioData);
+
+ const result = await sendJsonData('/transcribe', { audio: base64 })
+
+
const text = this.filterResult(result.text || "")
if (text) {
@@ -267,12 +268,29 @@ class MicrophoneInput {
console.error('Transcription error:', error);
toast('Transcription failed.', 'error');
} finally {
- URL.revokeObjectURL(audioUrl);
this.audioChunks = [];
this.status = Status.LISTENING;
}
}
+ convertBlobToBase64Wav(audioBlob) {
+ return new Promise((resolve, reject) => {
+ const reader = new FileReader();
+
+ // Read the Blob as a Data URL
+ reader.onloadend = () => {
+ const base64Data = reader.result.split(",")[1]; // Extract Base64 data
+ resolve(base64Data);
+ };
+
+ reader.onerror = (error) => {
+ reject(error);
+ };
+
+ reader.readAsDataURL(audioBlob); // Start reading the Blob
+ });
+ }
+
filterResult(text) {
text = text.trim()
let ok = false
@@ -370,12 +388,45 @@ class Speech {
// Remove emojis and create a new utterance
text = this.stripEmojis(text);
+ text = this.replaceURLs(text);
+ text = this.replaceGuids(text);
this.utterance = new SpeechSynthesisUtterance(text);
// Speak the new utterance
this.synth.speak(this.utterance);
}
+ replaceURLs(text) {
+ const urlRegex = /(\b(https?|ftp|file):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])|(\b(www\.)[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])|(\b[-A-Z0-9+&@#\/%?=~_|!:,.;]*\.(?:[A-Z]{2,})[-A-Z0-9+&@#\/%?=~_|])/ig; return text.replace(urlRegex, (url) => {
+ let text = url
+ // if contains ://, split by it
+ if (text.includes('://')) text = text.split('://')[1];
+ // if contains /, split by it
+ if (text.includes('/')) text = text.split('/')[0];
+
+ // if contains ., split by it
+ if (text.includes('.')) {
+ const doms = text.split('.')
+ //up to last two
+ return doms[doms.length - 2] + '.' + doms[doms.length - 1]
+ } else {
+ return text
+ }
+ });
+ }
+
+ replaceGuids(text) {
+ const guidRegex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g;
+ return text.replace(guidRegex, '');
+ }
+
+ replaceNonText(text) {
+ const nonTextRegex = /\w[^\w\s]*\w(?=\s|$)|[^\w\s]+/g;
+ return text.replace(nonTextRegex, (match) => {
+ return ``;
+ });
+ }
+
stop() {
if (this.isSpeaking()) {
this.synth.cancel();
diff --git a/webui/speech_browser.js b/webui/speech_browser.js
new file mode 100644
index 000000000..4ab46ab80
--- /dev/null
+++ b/webui/speech_browser.js
@@ -0,0 +1,394 @@
+import { pipeline, read_audio } from './[email protected]';
+import { updateChatInput, sendMessage } from './index.js';
+
+const microphoneButton = document.getElementById('microphone-button');
+let microphoneInput = null;
+let isProcessingClick = false;
+
+const Status = {
+ INACTIVE: 'inactive',
+ ACTIVATING: 'activating',
+ LISTENING: 'listening',
+ RECORDING: 'recording',
+ WAITING: 'waiting',
+ PROCESSING: 'processing'
+};
+
+class MicrophoneInput {
+ constructor(updateCallback, options = {}) {
+ this.mediaRecorder = null;
+ this.audioChunks = [];
+ this.lastChunk = [];
+ this.updateCallback = updateCallback;
+ this.messageSent = false;
+
+ // Audio analysis properties
+ this.audioContext = null;
+ this.mediaStreamSource = null;
+ this.analyserNode = null;
+ this._status = Status.INACTIVE;
+
+ // Timing properties
+ this.lastAudioTime = null;
+ this.waitingTimer = null;
+ this.silenceStartTime = null;
+ this.hasStartedRecording = false;
+ this.analysisFrame = null;
+
+ this.options = {
+ modelSize: 'tiny',
+ language: 'en',
+ silenceThreshold: 0.15,
+ silenceDuration: 1000,
+ waitingTimeout: 2000,
+ minSpeechDuration: 500,
+ ...options
+ };
+ }
+
+ get status() {
+ return this._status;
+ }
+
+ set status(newStatus) {
+ if (this._status === newStatus) return;
+
+ const oldStatus = this._status;
+ this._status = newStatus;
+ console.log(`Mic status changed from ${oldStatus} to ${newStatus}`);
+
+ // Update UI
+ microphoneButton.classList.remove(`mic-${oldStatus.toLowerCase()}`);
+ microphoneButton.classList.add(`mic-${newStatus.toLowerCase()}`);
+ microphoneButton.setAttribute('data-status', newStatus);
+
+ // Handle state-specific behaviors
+ this.handleStatusChange(oldStatus, newStatus);
+ }
+
+ handleStatusChange(oldStatus, newStatus) {
+
+ //last chunk kept only for transition to recording status
+ if (newStatus != Status.RECORDING) { this.lastChunk = null; }
+
+ switch (newStatus) {
+ case Status.INACTIVE:
+ this.handleInactiveState();
+ break;
+ case Status.LISTENING:
+ this.handleListeningState();
+ break;
+ case Status.RECORDING:
+ this.handleRecordingState();
+ break;
+ case Status.WAITING:
+ this.handleWaitingState();
+ break;
+ case Status.PROCESSING:
+ this.handleProcessingState();
+ break;
+ }
+ }
+
+ handleInactiveState() {
+ this.stopRecording();
+ this.stopAudioAnalysis();
+ if (this.waitingTimer) {
+ clearTimeout(this.waitingTimer);
+ this.waitingTimer = null;
+ }
+ }
+
+ handleListeningState() {
+ this.stopRecording();
+ this.audioChunks = [];
+ this.hasStartedRecording = false;
+ this.silenceStartTime = null;
+ this.lastAudioTime = null;
+ this.messageSent = false;
+ this.startAudioAnalysis();
+ }
+
+ handleRecordingState() {
+ if (!this.hasStartedRecording && this.mediaRecorder.state !== 'recording') {
+ this.hasStartedRecording = true;
+ this.mediaRecorder.start(1000);
+ console.log('Speech started');
+ }
+ if (this.waitingTimer) {
+ clearTimeout(this.waitingTimer);
+ this.waitingTimer = null;
+ }
+ }
+
+ handleWaitingState() {
+ // Don't stop recording during waiting state
+ this.waitingTimer = setTimeout(() => {
+ if (this.status === Status.WAITING) {
+ this.status = Status.PROCESSING;
+ }
+ }, this.options.waitingTimeout);
+ }
+
+ handleProcessingState() {
+ this.stopRecording();
+ this.process();
+ }
+
+ stopRecording() {
+ if (this.mediaRecorder?.state === 'recording') {
+ this.mediaRecorder.stop();
+ this.hasStartedRecording = false;
+ }
+ }
+
+ async initialize() {
+ try {
+ this.transcriber = await pipeline(
+ 'automatic-speech-recognition',
+ `Xenova/whisper-${this.options.modelSize}.${this.options.language}`
+ );
+
+ const stream = await navigator.mediaDevices.getUserMedia({
+ audio: {
+ echoCancellation: true,
+ noiseSuppression: true,
+ channelCount: 1
+ }
+ });
+
+ this.mediaRecorder = new MediaRecorder(stream);
+ this.mediaRecorder.ondataavailable = (event) => {
+ if (event.data.size > 0 &&
+ (this.status === Status.RECORDING || this.status === Status.WAITING)) {
+ if (this.lastChunk) {
+ this.audioChunks.push(this.lastChunk);
+ this.lastChunk = null;
+ }
+ this.audioChunks.push(event.data);
+ console.log('Audio chunk received, total chunks:', this.audioChunks.length);
+ }
+ else if (this.status === Status.LISTENING) {
+ this.lastChunk = event.data;
+ }
+ };
+
+ this.setupAudioAnalysis(stream);
+ return true;
+ } catch (error) {
+
+ console.error('Microphone initialization error:', error);
+ toast('Failed to access microphone. Please check permissions.', 'error');
+ return false;
+ }
+ }
+
+ setupAudioAnalysis(stream) {
+ this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
+ this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
+ this.analyserNode = this.audioContext.createAnalyser();
+ this.analyserNode.fftSize = 2048;
+ this.analyserNode.minDecibels = -90;
+ this.analyserNode.maxDecibels = -10;
+ this.analyserNode.smoothingTimeConstant = 0.85;
+ this.mediaStreamSource.connect(this.analyserNode);
+ }
+
+
+ startAudioAnalysis() {
+ const analyzeFrame = () => {
+ if (this.status === Status.INACTIVE) return;
+
+ const dataArray = new Uint8Array(this.analyserNode.fftSize);
+ this.analyserNode.getByteTimeDomainData(dataArray);
+
+ // Calculate RMS volume
+ let sum = 0;
+ for (let i = 0; i < dataArray.length; i++) {
+ const amplitude = (dataArray[i] - 128) / 128;
+ sum += amplitude * amplitude;
+ }
+ const rms = Math.sqrt(sum / dataArray.length);
+
+ const now = Date.now();
+
+ // Update status based on audio level
+ if (rms > this.options.silenceThreshold) {
+ this.lastAudioTime = now;
+ this.silenceStartTime = null;
+
+ if (this.status === Status.LISTENING || this.status === Status.WAITING) {
+ if (!speech.isSpeaking()) // TODO? a better way to ignore agent's voice?
+ this.status = Status.RECORDING;
+ }
+ } else if (this.status === Status.RECORDING) {
+ if (!this.silenceStartTime) {
+ this.silenceStartTime = now;
+ }
+
+ const silenceDuration = now - this.silenceStartTime;
+ if (silenceDuration >= this.options.silenceDuration) {
+ this.status = Status.WAITING;
+ }
+ }
+
+ this.analysisFrame = requestAnimationFrame(analyzeFrame);
+ };
+
+ this.analysisFrame = requestAnimationFrame(analyzeFrame);
+ }
+
+ stopAudioAnalysis() {
+ if (this.analysisFrame) {
+ cancelAnimationFrame(this.analysisFrame);
+ this.analysisFrame = null;
+ }
+ }
+
+ async process() {
+ if (this.audioChunks.length === 0) {
+ this.status = Status.LISTENING;
+ return;
+ }
+
+ const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });
+ const audioUrl = URL.createObjectURL(audioBlob);
+
+
+
+ try {
+ const samplingRate = 16000;
+ const audioData = await read_audio(audioUrl, samplingRate);
+ const result = await this.transcriber(audioData);
+ const text = this.filterResult(result.text || "")
+
+ if (text) {
+ console.log('Transcription:', result.text);
+ await this.updateCallback(result.text, true);
+ }
+ } catch (error) {
+ console.error('Transcription error:', error);
+ toast('Transcription failed.', 'error');
+ } finally {
+ URL.revokeObjectURL(audioUrl);
+ this.audioChunks = [];
+ this.status = Status.LISTENING;
+ }
+ }
+
+ filterResult(text) {
+ text = text.trim()
+ let ok = false
+ while (!ok) {
+ if (!text) break
+ if (text[0] === '{' && text[text.length - 1] === '}') break
+ if (text[0] === '(' && text[text.length - 1] === ')') break
+ if (text[0] === '[' && text[text.length - 1] === ']') break
+ ok = true
+ }
+ if (ok) return text
+ else console.log(`Discarding transcription: ${text}`)
+ }
+}
+
+
+
+// Initialize and handle click events
+async function initializeMicrophoneInput() {
+ microphoneInput = new MicrophoneInput(
+ async (text, isFinal) => {
+ if (isFinal) {
+ updateChatInput(text);
+ if (!microphoneInput.messageSent) {
+ microphoneInput.messageSent = true;
+ await sendMessage();
+ }
+ }
+ },
+ {
+ modelSize: 'tiny',
+ language: 'en',
+ silenceThreshold: 0.07,
+ silenceDuration: 1000,
+ waitingTimeout: 1500
+ }
+ );
+ microphoneInput.status = Status.ACTIVATING;
+
+ return await microphoneInput.initialize();
+}
+
+microphoneButton.addEventListener('click', async () => {
+ if (isProcessingClick) return;
+ isProcessingClick = true;
+
+ const hasPermission = await requestMicrophonePermission();
+ if (!hasPermission) return;
+
+ try {
+ if (!microphoneInput && !await initializeMicrophoneInput()) {
+ return;
+ }
+
+ // Simply toggle between INACTIVE and LISTENING states
+ microphoneInput.status =
+ (microphoneInput.status === Status.INACTIVE || microphoneInput.status === Status.ACTIVATING) ? Status.LISTENING : Status.INACTIVE;
+ } finally {
+ setTimeout(() => {
+ isProcessingClick = false;
+ }, 300);
+ }
+});
+
+// Some error handling for microphone input
+async function requestMicrophonePermission() {
+ try {
+ await navigator.mediaDevices.getUserMedia({ audio: true });
+ return true;
+ } catch (err) {
+ console.error('Error accessing microphone:', err);
+ toast('Microphone access denied. Please enable microphone access in your browser settings.', 'error');
+ return false;
+ }
+}
+
+
+class Speech {
+ constructor() {
+ this.synth = window.speechSynthesis;
+ this.utterance = null;
+ }
+
+ stripEmojis(str) {
+ return str
+ .replace(/([\u2700-\u27BF]|[\uE000-\uF8FF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDFFF]|[\u2011-\u26FF]|\uD83E[\uDD10-\uDDFF])/g, '')
+ .replace(/\s+/g, ' ')
+ .trim();
+ }
+
+ speak(text) {
+ console.log('Speaking:', text);
+ // Stop any current utterance
+ this.stop();
+
+ // Remove emojis and create a new utterance
+ text = this.stripEmojis(text);
+ this.utterance = new SpeechSynthesisUtterance(text);
+
+ // Speak the new utterance
+ this.synth.speak(this.utterance);
+ }
+
+ stop() {
+ if (this.isSpeaking()) {
+ this.synth.cancel();
+ }
+ }
+
+ isSpeaking() {
+ return this.synth?.speaking || false;
+ }
+}
+
+export const speech = new Speech();
+window.speech = speech
\ No newline at end of file