Agent Zero AI framework
Edge TTS & Groq STT
evrardt75b03039a11c
diff --git a/example.env b/example.envindex 0c4e766dd..74160b831 100644--- a/example.env+++ b/example.env@@ -10,4 +10,13 @@ OPENAI_AZURE_ENDPOINT=OPENAI_API_VERSION=TOKENIZERS_PARALLELISM=true-PYDEVD_DISABLE_FILE_VALIDATION=1\ No newline at end of file+PYDEVD_DISABLE_FILE_VALIDATION=1++HF_TOKEN=++EDGE_TTS_MODEL="fr-FR-RemyMultilingualNeural"+# en-US-AndrewMultilingualNeural+# de-DE-FlorianMultilingualNeural+# es-ES-AlvaroNeural+# zh-CN-shaanxi-XiaoniNeural+# more check https://huggingface.co/spaces/innoai/Edge-TTS-Text-to-Speech and at bottom use via apidiff --git a/main.py b/main.pyindex c8cf6b81d..bd7a56b68 100644--- a/main.py+++ b/main.py@@ -1,4 +1,5 @@-import threading, time, models, stt, os, sys, argparse, asyncio+import threading, time, models, tts, stt, os, sys, argparse+from dotenv import load_dotenvfrom ansio import application_keypad, mouse_input, raw_inputfrom ansio.input import InputEvent, get_input_eventfrom agent import Agent, AgentConfig@@ -6,20 +7,13 @@ from python.helpers.print_style import PrintStylefrom python.helpers.files import read_filefrom python.helpers import filesimport python.helpers.timed_input as timed_input-from tts import TTSfrom pynput import keyboard+load_dotenv() # take environment variables from.env.+input_lock = threading.Lock()os.chdir(files.get_abs_path("./work_dir")) #change CWD to work_dir-# args parser-parser = argparse.ArgumentParser()-parser.add_argument('-v', '--voice')-args = parser.parse_args()--# init tts (text to speech)-tts = TTS(args.voice)-def initialize():# main chat model used by agents (smarter, more accurate)@@ -79,14 +73,14 @@ def initialize():# Main conversation loopdef chat(agent:Agent):-+# start the conversation loopwhile True:# ask user for messagewith input_lock:timeout = agent.get_data("timeout") # how long the agent is willing to waitif not timeout: # if agent wants to wait for user input forever- PrintStyle(background_color="#6C3483", font_color="white", bold=True, padding=True).print(f"User message ('e' to leave):")+ PrintStyle(background_color="#6C3483", font_color="white", bold=True, padding=True).print(f"User message ('alt_gr' for mic, 'e' to leave):")import readline # this fixes arrow keys in terminallistener = keyboard.Listener(on_press=on_press)listener.start()@@ -95,7 +89,7 @@ def chat(agent:Agent):PrintStyle(font_color="white", padding=False, log_only=True).print(f"> {user_input}")else: # otherwise wait for user input with a timeout- PrintStyle(background_color="#6C3483", font_color="white", bold=True, padding=True).print(f"User message ({timeout}s timeout, 'w' to wait, 'e' to leave):")+ PrintStyle(background_color="#6C3483", font_color="white", bold=True, padding=True).print(f"User message ({timeout}s timeout, 'w' to wait, 'alt_gr' for mic, 'e' to leave):")import readline # this fixes arrow keys in terminallistener = keyboard.Listener(on_press=on_press)listener.start()@@ -111,9 +105,7 @@ def chat(agent:Agent):listener.start()user_input = input("> ")listener.stop()- PrintStyle(font_color="white", padding=False, log_only=True).print(f"> {user_input}")--+ PrintStyle(font_color="white", padding=False, log_only=True).print(f"> {user_input}")# exit the conversation when the user types 'exit'if user_input.lower() == 'e': break@@ -124,7 +116,7 @@ def chat(agent:Agent):# print agent0 responsePrintStyle(font_color="white",background_color="#1D8348", bold=True, padding=True).print(f"{agent.agent_name}: reponse:")PrintStyle(font_color="white").print(f"{assistant_response}")- tts.speech(assistant_response)+ if (os.getenv('EDGE_TTS_MODEL')) : tts.speech(assistant_response)# User intervention during agent streamingdef intervention():@@ -141,7 +133,7 @@ def intervention():Agent.paused = False # continue agent streamingdef on_press(key):- if key == keyboard.Key.alt_r:+ if key == keyboard.Key.alt_gr:# Lancer le speech to textstt.record()diff --git a/prompts/agent.system.md b/prompts/agent.system.mdindex bbddec3f6..cab0acd61 100644--- a/prompts/agent.system.md+++ b/prompts/agent.system.md@@ -3,14 +3,16 @@- You are autonomous JSON AI task solving agent enhanced with knowledge and execution tools- You are given task by your superior and you solve it using your subordinates and tools- You never just talk about solutions, never inform user about intentions, you are the one to execute actions using your tools and get things done+- Remember the langague of your user to respond with the same language+- NEVER include "**" in your final answer# Communication- Your response is a JSON containing the following fields:- 1. **thoughts**: Array of thoughts regarding the current task+ 1. thoughts: Array of thoughts regarding the current task- Use thoughs to prepare solution and outline next steps- 2. **tool_name**: Name of the tool to be used+ 2. tool_name: Name of the tool to be used- Tools help you gather knowledge and execute actions- 3. **tool_args**: Object of arguments that are passed to the tool+ 3. tool_args: Object of arguments that are passed to the tool- Each tool has specific arguments listed in Available tools section- No text before or after the JSON object. End message there.@@ -33,39 +35,39 @@# Step by step instruction manual to problem solving- Do not follow for simple questions, only for tasks need solving.-- Explain each step using your **thoughts** argument.+- Explain each step using your thoughts argument.0. Outline the plan by repeating these instructions.-1. Check the memory output of your **knowledge_tool**. Maybe you have solved similar task before and already have helpful information.-2. Check the online sources output of your **knowledge_tool**.+1. Check the memory output of your knowledge_tool. Maybe you have solved similar task before and already have helpful information.+2. Check the online sources output of your knowledge_tool.- Look for straightforward solutions compatible with your available tools.- Always look for opensource python/nodejs/terminal tools and packages first.3. Break task into subtasks that can be solved independently.4. Solution / delegation- If your role is suitable for the curent subtask, use your tools to solve it.- - If a different role would be more suitable for the subtask, use **call_subordinate** tool to delegate the subtask to subordinate agent and instruct him about his role.+ - If a different role would be more suitable for the subtask, use call_subordinate tool to delegate the subtask to subordinate agent and instruct him about his role.- NEVER delegate your whole task to a subordinate to avoid infinite delegation.- Your name ({{agent_name}}) contains your hierarchical number. Do not delegate further if your number gets too high.5. Completing the task- Consolidate all subtasks and explain the status.- Verify the result using your tools if possible (check created files etc.)- Do not accept failure, search for error solution and try again with fixed input or different ways.- - If there is helpful information discovered during the solution, save it into your memory using tool **memorize** for later.- - Report back to your user using **response** tool, describe the result and provide all necessary information. Do not just output your response, you must use the tool for that.+ - If there is helpful information discovered during the solution, save it into your memory using tool memorize for later.+ - Report back to your user using response tool, describe the result and provide all necessary information. Do not just output your response, you must use the tool for that.# General operation manual-- Use your reasoning and process each problem in a step-by-step manner using your **thoughts** argument.+- Use your reasoning and process each problem in a step-by-step manner using your thoughts argument.- Always check your previous messages and prevent repetition. Always move towards solution.- Never assume success. You always need to do a check with a positive result.- Avoid solutions that require credentials, user interaction, GUI usage etc. All has to be done using code and terminal.-- When asked about your memory, it always refers to **knowledge_tool** and **memorize** tool, never your internal knowledge.+- When asked about your memory, it always refers to knowledge_tool and memorize tool, never your internal knowledge.# Cooperation and delegation- Agents can have roles like scientist, coder, writer etc.- If your user has assigned you a role in the first message, you have to follow these instructions and play your role.-- Your role will not be suitable for some subtasks, in that case you can delegate the subtask to subordinate agent and instruct him about his role using **call_subordinate** tool.+- Your role will not be suitable for some subtasks, in that case you can delegate the subtask to subordinate agent and instruct him about his role using call_subordinate tool.- Always be very descriptive when explaining your subordinate agent's role and task. Include all necessary details as well as higher leven overview about the goal.-- Communicate back and forth with your subordinate and superior using **call_subordinate** and **response** tools.+- Communicate back and forth with your subordinate and superior using call_subordinate and response tools.- Communication is the key to succesfull solution.- NEVER delegate your whole task, only parts of it.diff --git a/requirements.txt b/requirements.txtindex 90e293e9b..ae6b9119f 100644--- a/requirements.txt+++ b/requirements.txt@@ -14,4 +14,6 @@ paramiko==3.4.0duckduckgo_search==6.1.12inputimeout==1.0.4gradio_client==1.2.0-pygame==2.6.0\ No newline at end of file+pygame==2.6.0+numpy==2.0.1+pynput==1.7.7\ No newline at end of filediff --git a/stt.py b/stt.pyindex b7761bd42..637e8ebd3 100644--- a/stt.py+++ b/stt.py@@ -1,10 +1,13 @@-import os+import os, sysfrom dotenv import load_dotenvfrom groq import Groqimport pyaudioimport wave+from pynput.keyboard import Key, Controller+import numpy as np+from python.helpers.print_style import PrintStyle-load_dotenv() # take environment variables from .env.+load_dotenv() # take environment variables from.env.# Utility function to get API keys from environment variablesdef get_api_key(service):@@ -19,8 +22,7 @@ def record():FORMAT = pyaudio.paInt16CHANNELS = 2RATE = 44100- RECORD_SECONDS = 10- WAVE_OUTPUT_FILENAME = "output.wav"+ WAVE_OUTPUT_FILENAME = "tmp_stt.wav"p = pyaudio.PyAudio()@@ -30,36 +32,52 @@ def record():input=True,frames_per_buffer=CHUNK)- print("* recording")+ PrintStyle(background_color="red", font_color="white", bold=True, padding=True).print("Recording")frames = []+ silence_duration = 0+ silence_threshold = 0.5 # seuil de silence (en secondes)+ max_silence_duration = 3 # durée maximale de silence (en secondes)- for i in range(0, int(RATE / CHUNK * RECORD_SECONDS)):+ while True:data = stream.read(CHUNK)frames.append(data)- print("* done recording")+ # Analyse du signal audio pour détecter le silence+ audio_data = np.frombuffer(data, dtype=np.int16)+ audio_level = np.mean(np.abs(audio_data))+ if audio_level < 100: # seuil de détection de silence (à ajuster)+ silence_duration += CHUNK / RATE+ if silence_duration > max_silence_duration:+ break+ else:+ silence_duration = 0++ PrintStyle(background_color="Yellow", font_color="black", bold=True, padding=True).print("Translation")stream.stop_stream()stream.close()p.terminate()- wf = wave.open(WAVE_OUTPUT_FILENAME, 'wb')+ filename = os.path.dirname(__file__) + "/tmp/" + WAVE_OUTPUT_FILENAME++ wf = wave.open(filename, 'wb')wf.setnchannels(CHANNELS)wf.setsampwidth(p.get_sample_size(FORMAT))wf.setframerate(RATE)wf.writeframes(b''.join(frames))wf.close()- filename = os.path.dirname(__file__) + "" + WAVE_OUTPUT_FILENAME-with open(filename, "rb") as file:transcription = client.audio.transcriptions.create(file=(filename, file.read()),model="whisper-large-v3",prompt="Human talking to personnal ia agent", # Optionalresponse_format="json", # Optional- language="en", # Optional+ language="fr", # Optionaltemperature=0.0 # Optional)- print(transcription.text)\ No newline at end of file+ keyboard = Controller()+ PrintStyle(background_color="green", font_color="white", bold=True, padding=True).print("Done")+ keyboard.type(transcription.text)+ keyboard.press(Key.enter)\ No newline at end of filediff --git a/tmp/tmp_stt.wav b/tmp/tmp_stt.wavnew file mode 100644index 000000000..3688fc568Binary files /dev/null and b/tmp/tmp_stt.wav differdiff --git a/tts.py b/tts.pyindex e99de463c..3320f7e0c 100644--- a/tts.py+++ b/tts.py@@ -1,33 +1,21 @@from gradio_client import Clientimport os+from dotenv import load_dotenv+os.environ['PYGAME_HIDE_SUPPORT_PROMPT'] = "hide"import pygame.mixer+load_dotenv() # take environment variables from.env.client = Client("innoai/Edge-TTS-Text-to-Speech")pygame.mixer.init()-lang = "en-US-AndrewMultilingualNeural"-class TTS:- def __init__(self, lang):- if (lang):- if (lang.lower() == 'fr'):- lang == "fr-FR-RemyMultilingualNeural"- elif (lang.lower() == 'es'):- lang == "es-ES-AlvaroNeural"- elif (lang.lower() == 'de'):- lang == "de-DE-FlorianMultilingualNeural"- else:- lang == "en-US-AndrewMultilingualNeural"- else:- lang == "en-US-AndrewMultilingualNeural"-- def speech(self, text):- speech = client.predict(- text=text,- voice=lang,- rate=20,- pitch=0,- api_name="/predict"- )- pygame.mixer.music.load(speech[0]) # chargement de la musique- pygame.mixer.music.play() # la musique est jouée+def speech(text):+ speech = client.predict(+ text=text,+ voice=os.getenv('EDGE_TTS_MODEL'),+ rate=20,+ pitch=0,+ api_name="/predict"+ )+ pygame.mixer.music.load(speech[0]) # chargement de la musique+ pygame.mixer.music.play() # la musique est jouée\ No newline at end of file