Skip to content

Agent Zero AI framework

Edge TTS & Groq STT

evrardt75b03039a11c
diff --git a/example.env b/example.env
index 0c4e766dd..74160b831 100644
--- a/example.env
+++ b/example.env
@@ -10,4 +10,13 @@ OPENAI_AZURE_ENDPOINT=
OPENAI_API_VERSION=
TOKENIZERS_PARALLELISM=true
-PYDEVD_DISABLE_FILE_VALIDATION=1
\ No newline at end of file
+PYDEVD_DISABLE_FILE_VALIDATION=1
+
+HF_TOKEN=
+
+EDGE_TTS_MODEL="fr-FR-RemyMultilingualNeural"
+# en-US-AndrewMultilingualNeural
+# de-DE-FlorianMultilingualNeural
+# es-ES-AlvaroNeural
+# zh-CN-shaanxi-XiaoniNeural
+# more check https://huggingface.co/spaces/innoai/Edge-TTS-Text-to-Speech and at bottom use via api
diff --git a/main.py b/main.py
index c8cf6b81d..bd7a56b68 100644
--- a/main.py
+++ b/main.py
@@ -1,4 +1,5 @@
-import threading, time, models, stt, os, sys, argparse, asyncio
+import threading, time, models, tts, stt, os, sys, argparse
+from dotenv import load_dotenv
from ansio import application_keypad, mouse_input, raw_input
from ansio.input import InputEvent, get_input_event
from agent import Agent, AgentConfig
@@ -6,20 +7,13 @@ from python.helpers.print_style import PrintStyle
from python.helpers.files import read_file
from python.helpers import files
import python.helpers.timed_input as timed_input
-from tts import TTS
from pynput import keyboard
+load_dotenv() # take environment variables from.env.
+
input_lock = threading.Lock()
os.chdir(files.get_abs_path("./work_dir")) #change CWD to work_dir
-# args parser
-parser = argparse.ArgumentParser()
-parser.add_argument('-v', '--voice')
-args = parser.parse_args()
-
-# init tts (text to speech)
-tts = TTS(args.voice)
-
def initialize():
# main chat model used by agents (smarter, more accurate)
@@ -79,14 +73,14 @@ def initialize():
# Main conversation loop
def chat(agent:Agent):
-
+
# start the conversation loop
while True:
# ask user for message
with input_lock:
timeout = agent.get_data("timeout") # how long the agent is willing to wait
if not timeout: # if agent wants to wait for user input forever
- PrintStyle(background_color="#6C3483", font_color="white", bold=True, padding=True).print(f"User message ('e' to leave):")
+ PrintStyle(background_color="#6C3483", font_color="white", bold=True, padding=True).print(f"User message ('alt_gr' for mic, 'e' to leave):")
import readline # this fixes arrow keys in terminal
listener = keyboard.Listener(on_press=on_press)
listener.start()
@@ -95,7 +89,7 @@ def chat(agent:Agent):
PrintStyle(font_color="white", padding=False, log_only=True).print(f"> {user_input}")
else: # otherwise wait for user input with a timeout
- PrintStyle(background_color="#6C3483", font_color="white", bold=True, padding=True).print(f"User message ({timeout}s timeout, 'w' to wait, 'e' to leave):")
+ PrintStyle(background_color="#6C3483", font_color="white", bold=True, padding=True).print(f"User message ({timeout}s timeout, 'w' to wait, 'alt_gr' for mic, 'e' to leave):")
import readline # this fixes arrow keys in terminal
listener = keyboard.Listener(on_press=on_press)
listener.start()
@@ -111,9 +105,7 @@ def chat(agent:Agent):
listener.start()
user_input = input("> ")
listener.stop()
- PrintStyle(font_color="white", padding=False, log_only=True).print(f"> {user_input}")
-
-
+ PrintStyle(font_color="white", padding=False, log_only=True).print(f"> {user_input}")
# exit the conversation when the user types 'exit'
if user_input.lower() == 'e': break
@@ -124,7 +116,7 @@ def chat(agent:Agent):
# print agent0 response
PrintStyle(font_color="white",background_color="#1D8348", bold=True, padding=True).print(f"{agent.agent_name}: reponse:")
PrintStyle(font_color="white").print(f"{assistant_response}")
- tts.speech(assistant_response)
+ if (os.getenv('EDGE_TTS_MODEL')) : tts.speech(assistant_response)
# User intervention during agent streaming
def intervention():
@@ -141,7 +133,7 @@ def intervention():
Agent.paused = False # continue agent streaming
def on_press(key):
- if key == keyboard.Key.alt_r:
+ if key == keyboard.Key.alt_gr:
# Lancer le speech to text
stt.record()
diff --git a/prompts/agent.system.md b/prompts/agent.system.md
index bbddec3f6..cab0acd61 100644
--- a/prompts/agent.system.md
+++ b/prompts/agent.system.md
@@ -3,14 +3,16 @@
- You are autonomous JSON AI task solving agent enhanced with knowledge and execution tools
- You are given task by your superior and you solve it using your subordinates and tools
- You never just talk about solutions, never inform user about intentions, you are the one to execute actions using your tools and get things done
+- Remember the langague of your user to respond with the same language
+- NEVER include "**" in your final answer
# Communication
- Your response is a JSON containing the following fields:
- 1. **thoughts**: Array of thoughts regarding the current task
+ 1. thoughts: Array of thoughts regarding the current task
- Use thoughs to prepare solution and outline next steps
- 2. **tool_name**: Name of the tool to be used
+ 2. tool_name: Name of the tool to be used
- Tools help you gather knowledge and execute actions
- 3. **tool_args**: Object of arguments that are passed to the tool
+ 3. tool_args: Object of arguments that are passed to the tool
- Each tool has specific arguments listed in Available tools section
- No text before or after the JSON object. End message there.
@@ -33,39 +35,39 @@
# Step by step instruction manual to problem solving
- Do not follow for simple questions, only for tasks need solving.
-- Explain each step using your **thoughts** argument.
+- Explain each step using your thoughts argument.
0. Outline the plan by repeating these instructions.
-1. Check the memory output of your **knowledge_tool**. Maybe you have solved similar task before and already have helpful information.
-2. Check the online sources output of your **knowledge_tool**.
+1. Check the memory output of your knowledge_tool. Maybe you have solved similar task before and already have helpful information.
+2. Check the online sources output of your knowledge_tool.
- Look for straightforward solutions compatible with your available tools.
- Always look for opensource python/nodejs/terminal tools and packages first.
3. Break task into subtasks that can be solved independently.
4. Solution / delegation
- If your role is suitable for the curent subtask, use your tools to solve it.
- - If a different role would be more suitable for the subtask, use **call_subordinate** tool to delegate the subtask to subordinate agent and instruct him about his role.
+ - If a different role would be more suitable for the subtask, use call_subordinate tool to delegate the subtask to subordinate agent and instruct him about his role.
- NEVER delegate your whole task to a subordinate to avoid infinite delegation.
- Your name ({{agent_name}}) contains your hierarchical number. Do not delegate further if your number gets too high.
5. Completing the task
- Consolidate all subtasks and explain the status.
- Verify the result using your tools if possible (check created files etc.)
- Do not accept failure, search for error solution and try again with fixed input or different ways.
- - If there is helpful information discovered during the solution, save it into your memory using tool **memorize** for later.
- - Report back to your user using **response** tool, describe the result and provide all necessary information. Do not just output your response, you must use the tool for that.
+ - If there is helpful information discovered during the solution, save it into your memory using tool memorize for later.
+ - Report back to your user using response tool, describe the result and provide all necessary information. Do not just output your response, you must use the tool for that.
# General operation manual
-- Use your reasoning and process each problem in a step-by-step manner using your **thoughts** argument.
+- Use your reasoning and process each problem in a step-by-step manner using your thoughts argument.
- Always check your previous messages and prevent repetition. Always move towards solution.
- Never assume success. You always need to do a check with a positive result.
- Avoid solutions that require credentials, user interaction, GUI usage etc. All has to be done using code and terminal.
-- When asked about your memory, it always refers to **knowledge_tool** and **memorize** tool, never your internal knowledge.
+- When asked about your memory, it always refers to knowledge_tool and memorize tool, never your internal knowledge.
# Cooperation and delegation
- Agents can have roles like scientist, coder, writer etc.
- If your user has assigned you a role in the first message, you have to follow these instructions and play your role.
-- Your role will not be suitable for some subtasks, in that case you can delegate the subtask to subordinate agent and instruct him about his role using **call_subordinate** tool.
+- Your role will not be suitable for some subtasks, in that case you can delegate the subtask to subordinate agent and instruct him about his role using call_subordinate tool.
- Always be very descriptive when explaining your subordinate agent's role and task. Include all necessary details as well as higher leven overview about the goal.
-- Communicate back and forth with your subordinate and superior using **call_subordinate** and **response** tools.
+- Communicate back and forth with your subordinate and superior using call_subordinate and response tools.
- Communication is the key to succesfull solution.
- NEVER delegate your whole task, only parts of it.
diff --git a/requirements.txt b/requirements.txt
index 90e293e9b..ae6b9119f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,4 +14,6 @@ paramiko==3.4.0
duckduckgo_search==6.1.12
inputimeout==1.0.4
gradio_client==1.2.0
-pygame==2.6.0
\ No newline at end of file
+pygame==2.6.0
+numpy==2.0.1
+pynput==1.7.7
\ No newline at end of file
diff --git a/stt.py b/stt.py
index b7761bd42..637e8ebd3 100644
--- a/stt.py
+++ b/stt.py
@@ -1,10 +1,13 @@
-import os
+import os, sys
from dotenv import load_dotenv
from groq import Groq
import pyaudio
import wave
+from pynput.keyboard import Key, Controller
+import numpy as np
+from python.helpers.print_style import PrintStyle
-load_dotenv() # take environment variables from .env.
+load_dotenv() # take environment variables from.env.
# Utility function to get API keys from environment variables
def get_api_key(service):
@@ -19,8 +22,7 @@ def record():
FORMAT = pyaudio.paInt16
CHANNELS = 2
RATE = 44100
- RECORD_SECONDS = 10
- WAVE_OUTPUT_FILENAME = "output.wav"
+ WAVE_OUTPUT_FILENAME = "tmp_stt.wav"
p = pyaudio.PyAudio()
@@ -30,36 +32,52 @@ def record():
input=True,
frames_per_buffer=CHUNK)
- print("* recording")
+ PrintStyle(background_color="red", font_color="white", bold=True, padding=True).print("Recording")
frames = []
+ silence_duration = 0
+ silence_threshold = 0.5 # seuil de silence (en secondes)
+ max_silence_duration = 3 # durée maximale de silence (en secondes)
- for i in range(0, int(RATE / CHUNK * RECORD_SECONDS)):
+ while True:
data = stream.read(CHUNK)
frames.append(data)
- print("* done recording")
+ # Analyse du signal audio pour détecter le silence
+ audio_data = np.frombuffer(data, dtype=np.int16)
+ audio_level = np.mean(np.abs(audio_data))
+ if audio_level < 100: # seuil de détection de silence (à ajuster)
+ silence_duration += CHUNK / RATE
+ if silence_duration > max_silence_duration:
+ break
+ else:
+ silence_duration = 0
+
+ PrintStyle(background_color="Yellow", font_color="black", bold=True, padding=True).print("Translation")
stream.stop_stream()
stream.close()
p.terminate()
- wf = wave.open(WAVE_OUTPUT_FILENAME, 'wb')
+ filename = os.path.dirname(__file__) + "/tmp/" + WAVE_OUTPUT_FILENAME
+
+ wf = wave.open(filename, 'wb')
wf.setnchannels(CHANNELS)
wf.setsampwidth(p.get_sample_size(FORMAT))
wf.setframerate(RATE)
wf.writeframes(b''.join(frames))
wf.close()
- filename = os.path.dirname(__file__) + "" + WAVE_OUTPUT_FILENAME
-
with open(filename, "rb") as file:
transcription = client.audio.transcriptions.create(
file=(filename, file.read()),
model="whisper-large-v3",
prompt="Human talking to personnal ia agent", # Optional
response_format="json", # Optional
- language="en", # Optional
+ language="fr", # Optional
temperature=0.0 # Optional
)
- print(transcription.text)
\ No newline at end of file
+ keyboard = Controller()
+ PrintStyle(background_color="green", font_color="white", bold=True, padding=True).print("Done")
+ keyboard.type(transcription.text)
+ keyboard.press(Key.enter)
\ No newline at end of file
diff --git a/tmp/tmp_stt.wav b/tmp/tmp_stt.wav
new file mode 100644
index 000000000..3688fc568
Binary files /dev/null and b/tmp/tmp_stt.wav differ
diff --git a/tts.py b/tts.py
index e99de463c..3320f7e0c 100644
--- a/tts.py
+++ b/tts.py
@@ -1,33 +1,21 @@
from gradio_client import Client
import os
+from dotenv import load_dotenv
+os.environ['PYGAME_HIDE_SUPPORT_PROMPT'] = "hide"
import pygame.mixer
+load_dotenv() # take environment variables from.env.
client = Client("innoai/Edge-TTS-Text-to-Speech")
pygame.mixer.init()
-lang = "en-US-AndrewMultilingualNeural"
-class TTS:
- def __init__(self, lang):
- if (lang):
- if (lang.lower() == 'fr'):
- lang == "fr-FR-RemyMultilingualNeural"
- elif (lang.lower() == 'es'):
- lang == "es-ES-AlvaroNeural"
- elif (lang.lower() == 'de'):
- lang == "de-DE-FlorianMultilingualNeural"
- else:
- lang == "en-US-AndrewMultilingualNeural"
- else:
- lang == "en-US-AndrewMultilingualNeural"
-
- def speech(self, text):
- speech = client.predict(
- text=text,
- voice=lang,
- rate=20,
- pitch=0,
- api_name="/predict"
- )
- pygame.mixer.music.load(speech[0]) # chargement de la musique
- pygame.mixer.music.play() # la musique est jouée
+def speech(text):
+ speech = client.predict(
+ text=text,
+ voice=os.getenv('EDGE_TTS_MODEL'),
+ rate=20,
+ pitch=0,
+ api_name="/predict"
+ )
+ pygame.mixer.music.load(speech[0]) # chargement de la musique
+ pygame.mixer.music.play() # la musique est jouée
\ No newline at end of file