Agent Zero AI framework
TTS prototype
frdel92d94b4d86ef
diff --git a/webui/index.css b/webui/index.cssindex fde8a6c4e..cb3848478 100644--- a/webui/index.css+++ b/webui/index.css@@ -446,6 +446,8 @@ pre {padding-bottom: 0;*/padding: var(--spacing-sm) var(--spacing-md);padding-bottom: 0;+ display: flex;+ justify-content: space-between;}#progress-bar-h {diff --git a/webui/index.html b/webui/index.htmlindex 309670d60..131467b6e 100644--- a/webui/index.html+++ b/webui/index.html@@ -9,7 +9,7 @@<link rel="stylesheet" href="toast.css"><link rel="stylesheet" href="settings.css"><link rel="stylesheet" href="speech.css">-+<script>window.safeCall = function (name, ...args) {if (window[name]) window[name](...args)@@ -21,7 +21,8 @@<!-- KaTeX javascript --><script defer src="https://cdn.jsdelivr.net/npm/[email protected]/dist/katex.min.js" crossorigin="anonymous"></script>- <script defer src="https://cdn.jsdelivr.net/npm/[email protected]/dist/contrib/auto-render.min.js" crossorigin="anonymous"></script>+ <script defer src="https://cdn.jsdelivr.net/npm/[email protected]/dist/contrib/auto-render.min.js"+ crossorigin="anonymous"></script><script defer src="https://cdn.jsdelivr.net/npm/@alpinejs/[email protected]/dist/cdn.min.js"></script><script defer src="https://cdn.jsdelivr.net/npm/[email protected]/dist/cdn.min.js"></script>@@ -39,23 +40,24 @@<button id="toggle-sidebar" class="toggle-sidebar-button" aria-label="Toggle Sidebar" aria-expanded="false"><span aria-hidden="true"><!-- Hamburger Icon -->- <svg id="sidebar-hamburger-svg" xmlns="http://www.w3.org/2000/svg" width="22" height="22" viewBox="0 0 24 24" fill="CurrentColor">- <path d="M3 13h18v-2H3v2zm0 4h18v-2H3v2zm0-8h18V7H3v2z"></path>+ <svg id="sidebar-hamburger-svg" xmlns="http://www.w3.org/2000/svg" width="22" height="22"+ viewBox="0 0 24 24" fill="CurrentColor">+ <path d="M3 13h18v-2H3v2zm0 4h18v-2H3v2zm0-8h18V7H3v2z"></path></svg></span></button>-- <div id="logo-container">++ <div id="logo-container"><a href="https://github.com/frdel/agent-zero" target="_blank" rel="noopener noreferrer"><img src="splash.jpg" alt="a0" width="22" height="22"></a>+ </div></div>- </div>-- <div id="left-panel" class="panel">- <!--Sidebar upper elements-->- <div class="left-panel-top">- <div class="config-section" id="status-section" x-data="{ connected: true }">++ <div id="left-panel" class="panel">+ <!--Sidebar upper elements-->+ <div class="left-panel-top">+ <div class="config-section" id="status-section" x-data="{ connected: true }"><h3>Status</h3><h4 class="connected" x-show="connected">✔ Connected</h4><h4 class="disconnected" x-show="!connected">✘ Disconnected</h4>@@ -72,85 +74,95 @@</div><!-- Chats List --><div class="config-section" id="chats-section" x-data="{ contexts: [], selected: '' }"- x-show="contexts.length > 0">- <h3>Chats</h3>- <div class="chats-list-container">- <ul class="config-list">- <template x-for="context in contexts">- <li>- <span :class="{'chat-list-button': true, 'font-bold': context.id === selected}"- @click="selected = context.id; selectChat(context.id)">- Chat #<span x-text="context.no"></span>- </span>- <button class="edit-button" @click="killChat(context.id)">X</button>- </li>- </template>- </ul>+ x-show="contexts.length > 0">+ <h3>Chats</h3>+ <div class="chats-list-container">+ <ul class="config-list">+ <template x-for="context in contexts">+ <li>+ <span :class="{'chat-list-button': true, 'font-bold': context.id === selected}"+ @click="selected = context.id; selectChat(context.id)">+ Chat #<span x-text="context.no"></span>+ </span>+ <button class="edit-button" @click="killChat(context.id)">X</button>+ </li>+ </template>+ </ul>+ </div></div></div>- </div>- <!--Sidebar lower elements-->- <div class="left-panel-bottom">- <!-- Preferences -->- <div class="pref-section" x-data="{ prefOpen: true }">- <span><h3 class="pref-header" @click="prefOpen = !prefOpen">- Preferences- <svg class="arrow-icon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"- fill="none" stroke="currentColor" stroke-width="2" width="16" height="16"- :class="{'rotated': !prefOpen}">- <path d="M8 4l8 8-8 8"/>- </svg>- </h3>- <ul class="config-list" x-show="prefOpen" x-collapse x-transition>- <!-- Preferences Items -->- <li x-data="{ autoScroll: true }">- <span>Autoscroll</span>- <label class="switch">- <input id="auto-scroll-switch" type="checkbox" x-model="autoScroll"- x-effect="window.safeCall('toggleAutoScroll',autoScroll)">- <span class="slider"></span>- </label>- </li>- <li x-data="{ darkMode: localStorage.getItem('darkMode') != 'false' }"- x-init="$watch('darkMode', val => toggleDarkMode(val))">- <span class="switch-label">Dark mode</span>- <label class="switch">- <input type="checkbox" x-model="darkMode">- <span class="slider"></span>- </label>- </li>- <li x-data="{ showThoughts: true }">- <span>Show thoughts</span>- <label class="switch">- <input type="checkbox" x-model="showThoughts"- x-effect="window.safeCall('toggleThoughts',showThoughts)">- <span class="slider"></span>- </label>- </li>- <li x-data="{ showJson: false }">- <span>Show JSON</span>- <label class="switch">- <input type="checkbox" x-model="showJson" x-effect="window.safeCall('toggleJson',showJson)">- <span class="slider"></span>- </label>- </li>- <li x-data="{ showUtils: false }">- <span>Show utility messages</span>- <label class="switch">- <input type="checkbox" x-model="showUtils"- x-effect="window.safeCall('toggleUtils',showUtils)">- <span class="slider"></span>- </label>- </li>- </ul>- </span>- </div>- <!-- Version Info -->- <div class="version-info">- <span id="a0version">Agent Zero 0.7.2<br>built on 2024-11-04</span>+ <!--Sidebar lower elements-->+ <div class="left-panel-bottom">+ <!-- Preferences -->+ <div class="pref-section" x-data="{ prefOpen: true }">+ <span>+ <h3 class="pref-header" @click="prefOpen = !prefOpen">+ Preferences+ <svg class="arrow-icon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"+ stroke="currentColor" stroke-width="2" width="16" height="16"+ :class="{'rotated': !prefOpen}">+ <path d="M8 4l8 8-8 8" />+ </svg>+ </h3>+ <ul class="config-list" x-show="prefOpen" x-collapse x-transition>+ <!-- Preferences Items -->+ <li x-data="{ autoScroll: true }">+ <span>Autoscroll</span>+ <label class="switch">+ <input id="auto-scroll-switch" type="checkbox" x-model="autoScroll"+ x-effect="window.safeCall('toggleAutoScroll',autoScroll)">+ <span class="slider"></span>+ </label>+ </li>+ <li x-data="{ darkMode: localStorage.getItem('darkMode') != 'false' }"+ x-init="$watch('darkMode', val => toggleDarkMode(val))">+ <span class="switch-label">Dark mode</span>+ <label class="switch">+ <input type="checkbox" x-model="darkMode">+ <span class="slider"></span>+ </label>+ </li>+ <li x-data="{ speech: localStorage.getItem('speech') == 'true' }"+ x-init="$watch('speech', val => toggleSpeech(val))">+ <span class="switch-label">Speech</span>+ <label class="switch">+ <input type="checkbox" x-model="speech">+ <span class="slider"></span>+ </label>+ </li>+ <li x-data="{ showThoughts: true }">+ <span>Show thoughts</span>+ <label class="switch">+ <input type="checkbox" x-model="showThoughts"+ x-effect="window.safeCall('toggleThoughts',showThoughts)">+ <span class="slider"></span>+ </label>+ </li>+ <li x-data="{ showJson: false }">+ <span>Show JSON</span>+ <label class="switch">+ <input type="checkbox" x-model="showJson"+ x-effect="window.safeCall('toggleJson',showJson)">+ <span class="slider"></span>+ </label>+ </li>+ <li x-data="{ showUtils: false }">+ <span>Show utility messages</span>+ <label class="switch">+ <input type="checkbox" x-model="showUtils"+ x-effect="window.safeCall('toggleUtils',showUtils)">+ <span class="slider"></span>+ </label>+ </li>+ </ul>+ </span>+ </div>+ <!-- Version Info -->+ <div class="version-info">+ <span id="a0version">Agent Zero 0.7.2<br>built on 2024-11-04</span>+ </div></div></div>- </div><div id="right-panel" class="panel"><!--Chat--><div id="time-date"></div>@@ -161,8 +173,16 @@<button class="toast__copy">Copy</button><button class="toast__close">Close</button></div>- <div id="progress-bar-box">- <h4 id="progress-bar-h"><span id="progress-bar-i">|></span><span id="progress-bar"></span></h4>+ <div id="progress-bar-box" x-data="{ isSpeaking: false }" x-init="+ // Watch the speech synthesis status to update isSpeaking+ setInterval(() => isSpeaking = window.speech.isSpeaking(), 100)+ ">+ <h4 id="progress-bar-h">+ <span id="progress-bar-i">|></span><span id="progress-bar"></span>+ </h4>+ <h4 id="progress-bar-stop-speech" x-show="isSpeaking">+ <span id="stop-speech" @click="window.speech.stop()" style="cursor: pointer">Stop Speech</span>+ </h4></div><div id="input-section" x-data="{paused: false,@@ -197,8 +217,8 @@<template x-for="(image, index) in attachments" :key="index"><div class="preview-item"><img :src="image.url" alt="Preview">- <button @click="attachments.splice(index, 1); hasAttachments = attachments.length > 0"- class="remove-image">×</button>+ <button @click="attachments.splice(index, 1); hasAttachments = attachments.length > 0"+ class="remove-image">×</button></div></template></div>@@ -207,65 +227,80 @@<div class="input-row"><!-- Attachment icon with tooltip --><div class="attachment-wrapper" x-data="{ showTooltip: false }">- <label for="file-input" class="attachment-icon"- @mouseover="showTooltip = true"- @mouseleave="showTooltip = false">- <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="currentColor">- <path d="M16.5 6v11.5c0 2.21-1.79 4-4 4s-4-1.79-4-4V5c0-1.38 1.12-2.5 2.5-2.5s2.5 1.12 2.5 2.5v10.5c0 .55-.45 1-1 1s-1-.45-1-1V6H10v9.5c0 1.38 1.12 2.5 2.5 2.5s2.5-1.12 2.5-2.5V5c0-2.21-1.79-4-4-4S7 2.79 7 5v12.5c0 3.04 2.46 5.5 5.5 5.5s5.5-2.46 5.5-5.5V6h-1.5z"/>+ <label for="file-input" class="attachment-icon" @mouseover="showTooltip = true"+ @mouseleave="showTooltip = false">+ <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24"+ fill="currentColor">+ <path+ d="M16.5 6v11.5c0 2.21-1.79 4-4 4s-4-1.79-4-4V5c0-1.38 1.12-2.5 2.5-2.5s2.5 1.12 2.5 2.5v10.5c0 .55-.45 1-1 1s-1-.45-1-1V6H10v9.5c0 1.38 1.12 2.5 2.5 2.5s2.5-1.12 2.5-2.5V5c0-2.21-1.79-4-4-4S7 2.79 7 5v12.5c0 3.04 2.46 5.5 5.5 5.5s5.5-2.46 5.5-5.5V6h-1.5z" /></svg></label>- <input type="file" id="file-input"- accept="png, jpg, jpeg, txt, pdf, csv, html, json, md"- multiple- style="display: none"- @change="handleFileUpload($event)">+ <input type="file" id="file-input" accept="png, jpg, jpeg, txt, pdf, csv, html, json, md"+ multiple style="display: none" @change="handleFileUpload($event)"><div x-show="showTooltip" class="tooltip">Limit: 4 attachments per message</div></div>-+<!-- Text input --><textarea id="chat-input" placeholder="Type your message here..." rows="1"></textarea>-+<div id="chat-buttons-wrapper">- <!-- Send button -->- <button class="chat-button" id="send-button" aria-label="Send message">- <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">- <path d="M25 20 L75 50 L25 80" fill="none" stroke="currentColor" stroke-width="15"></path>- </svg>- </button>-- <!-- Microphone button -->- <button class="chat-button mic-inactive" id="microphone-button" aria-label="Start/Stop recording">- <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 18" fill="currentColor" width="24" height="24">- <path d="m8,12c1.66,0,3-1.34,3-3V3c0-1.66-1.34-3-3-3s-3,1.34-3,3v6c0,1.66,1.34,3,3,3Zm-1,1.9c-2.7-.4-4.8-2.6-5-5.4H0c.2,3.8,3.1,6.9,7,7.5v2h2v-2c3.9-.6,6.8-3.7,7-7.5h-2c-.2,2.8-2.3,5-5,5.4h-2Z"/>- </svg>+ <!-- Send button -->+ <button class="chat-button" id="send-button" aria-label="Send message">+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">+ <path d="M25 20 L75 50 L25 80" fill="none" stroke="currentColor" stroke-width="15">+ </path>+ </svg>+ </button>- </button>+ <!-- Microphone button -->+ <button class="chat-button mic-inactive" id="microphone-button"+ aria-label="Start/Stop recording">+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 18" fill="currentColor" width="24"+ height="24">+ <path+ d="m8,12c1.66,0,3-1.34,3-3V3c0-1.66-1.34-3-3-3s-3,1.34-3,3v6c0,1.66,1.34,3,3,3Zm-1,1.9c-2.7-.4-4.8-2.6-5-5.4H0c.2,3.8,3.1,6.9,7,7.5v2h2v-2c3.9-.6,6.8-3.7,7-7.5h-2c-.2,2.8-2.3,5-5,5.4h-2Z" />+ </svg>++ </button></div></div>-+<!-- Bottom row with text buttons --><div class="text-buttons-row">- <button class="text-button" @click="loadKnowledge()"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M3 16.5v2.25A2.25 2.25 0 0 0 5.25 21h13.5A2.25 2.25 0 0 0 21 18.75V16.5m-13.5-9L12 3m0 0 4.5 4.5M12 3v13.5"></path></svg>Import knowledge</button>+ <button class="text-button" @click="loadKnowledge()"><svg xmlns="http://www.w3.org/2000/svg"+ fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor">+ <path stroke-linecap="round" stroke-linejoin="round"+ d="M3 16.5v2.25A2.25 2.25 0 0 0 5.25 21h13.5A2.25 2.25 0 0 0 21 18.75V16.5m-13.5-9L12 3m0 0 4.5 4.5M12 3v13.5">+ </path>+ </svg>Import knowledge</button><button class="text-button" @click="workDirModalProxy.openModal()"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 123.37 92.59">- <path d="m5.72,11.5l-3.93,8.73h119.77s-3.96-8.73-3.96-8.73h-60.03c-1.59,0-2.88-1.29-2.88-2.88V1.75H13.72v6.87c0,1.59-1.29,2.88-2.88,2.88h-5.12Z" fill="none" stroke="currentColor" stroke-linejoin="round" stroke-width="5"></path>- <path d="m6.38,20.23H1.75l7.03,67.03c.11,1.07.55,2.02,1.2,2.69.55.55,1.28.89,2.11.89h97.1c.82,0,1.51-.33,2.05-.87.68-.68,1.13-1.67,1.28-2.79l9.1-66.94H6.38Z" fill="none" stroke="currentColor" stroke-linejoin="round" stroke-width="5"></path>- </svg>work_dir Browser</button>-+ <path+ d="m5.72,11.5l-3.93,8.73h119.77s-3.96-8.73-3.96-8.73h-60.03c-1.59,0-2.88-1.29-2.88-2.88V1.75H13.72v6.87c0,1.59-1.29,2.88-2.88,2.88h-5.12Z"+ fill="none" stroke="currentColor" stroke-linejoin="round" stroke-width="5"></path>+ <path+ d="m6.38,20.23H1.75l7.03,67.03c.11,1.07.55,2.02,1.2,2.69.55.55,1.28.89,2.11.89h97.1c.82,0,1.51-.33,2.05-.87.68-.68,1.13-1.67,1.28-2.79l9.1-66.94H6.38Z"+ fill="none" stroke="currentColor" stroke-linejoin="round" stroke-width="5"></path>+ </svg>work_dir Browser</button>+<button class="text-button" @click="pauseAgent(!paused)">- <!-- Dynamic path that switches between pause and play icons -->- <template x-if="!paused">- <svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" width="14" height="14">- <path d="M6 19h4V5H6v14zm8-14v14h4V5h-4z"></path></svg>- </template>- <template x-if="paused">- <svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" width="14" height="14">- <path d="M8 5v14l11-7z"></path></svg>- </template>+ <!-- Dynamic path that switches between pause and play icons -->+ <template x-if="!paused">+ <svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" viewBox="0 0 24 24"+ stroke-width="1.5" stroke="currentColor" width="14" height="14">+ <path d="M6 19h4V5H6v14zm8-14v14h4V5h-4z"></path>+ </svg>+ </template>+ <template x-if="paused">+ <svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" viewBox="0 0 24 24"+ stroke-width="1.5" stroke="currentColor" width="14" height="14">+ <path d="M8 5v14l11-7z"></path>+ </svg>+ </template></svg><span x-text="paused ? 'Resume Agent' : 'Pause Agent'"></span></button>@@ -275,8 +310,7 @@</div><div id="settingsModal" x-data="settingsModalProxy"><template x-teleport="body">- <div x-show="isOpen" class="modal-overlay" @click.self="handleCancel()"- x-transition>+ <div x-show="isOpen" class="modal-overlay" @click.self="handleCancel()" x-transition><div class="modal-container"><div class="modal-header"><h2 x-text="settings.title"></h2>@@ -369,10 +403,10 @@<div class="modal-footer"><div id="buttons-container">- <template x-for="button in settings.buttons" :key="button.id">- <button :class="button.classes" @click="handleButton(button.id)"- x-text="button.title"></button>- </template>+ <template x-for="button in settings.buttons" :key="button.id">+ <button :class="button.classes" @click="handleButton(button.id)"+ x-text="button.title"></button>+ </template></div></div></div>diff --git a/webui/index.js b/webui/index.jsindex 2b87b9875..e0e2fe15b 100644--- a/webui/index.js+++ b/webui/index.js@@ -1,4 +1,5 @@import * as msgs from "./messages.js"+import { speech } from "./speech.js";const leftPanel = document.getElementById('left-panel');const rightPanel = document.getElementById('right-panel');@@ -59,7 +60,7 @@ function setupSidebarToggle() {// Make sure to call this functiondocument.addEventListener('DOMContentLoaded', setupSidebarToggle);- // index.js+// index.jsexport async function sendMessage() {try {const message = chatInput.value.trim();@@ -77,7 +78,7 @@ export async function sendMessage() {...attachment,url: URL.createObjectURL(attachment.file)}));-+// Only render if there's text content or it's an image-only messagesetMessage(messageId, 'user', '', message, false, {attachments: attachmentsWithUrls@@ -98,10 +99,10 @@ export async function sendMessage() {});} else {// For text-only messages, let polling handle the rendering- const data = {- text: message,+ const data = {+ text: message,context,- message_id: messageId+ message_id: messageId};response = await fetch('/msg', {method: 'POST',@@ -149,16 +150,16 @@ sendButton.addEventListener('click', sendMessage);export function updateChatInput(text) {console.log('updateChatInput called with:', text);-+// Append text with proper spacingconst currentValue = chatInput.value;const needsSpace = currentValue.length > 0 && !currentValue.endsWith(' ');chatInput.value = currentValue + (needsSpace ? ' ' : '') + text + ' ';-+// Adjust height and trigger input eventadjustTextareaHeight();chatInput.dispatchEvent(new Event('input'));-+console.log('Updated chat input value:', chatInput.value);}@@ -235,7 +236,7 @@ async function handleFileUpload(event) {}-window.loadKnowledge = async function() {+window.loadKnowledge = async function () {const input = document.createElement('input');input.type = 'file';input.accept = '.txt,.pdf,.csv,.html,.json,.md';@@ -289,17 +290,17 @@ const workDirModalProxy = {};// Make the proxy available globally-window.workDirModalProxy = workDirModalProxy;+window.workDirModalProxy = workDirModalProxy;// Ensure correct setup for Alpine.js x-data.-window.workDirModal = function() {+window.workDirModal = function () {return workDirModalProxy; // Returns the proxy object for the Work Dir modal}document.addEventListener('alpine:init', () => {// Make workDirModalProxy available as an Alpine component/store- Alpine.data('workDirModal', workDirModal);+ Alpine.data('workDirModal', workDirModal);});@@ -334,6 +335,7 @@ function generateGUID() {let lastLogVersion = 0;let lastLogGuid = ""+let lastSpokenNo = 0async function poll() {let updated = false@@ -356,6 +358,7 @@ async function poll() {for (const log of response.logs) {setMessage(log.no, log.type, log.heading, log.content, log.temp, log.kvps);}+ afterMessagesUpdate(response.logs)}updateProgress(response.log_progress)@@ -383,6 +386,26 @@ async function poll() {return updated}+function afterMessagesUpdate(logs) {+ if(localStorage.getItem('speech') == 'true') {+ speakMessages(logs)+ }+}++function speakMessages(logs) {+ // log.no, log.type, log.heading, log.content+ for (let i = logs.length - 1; i >= 0; i--) {+ const log = logs[i]+ if (log.type == "response") {+ if (log.no > lastSpokenNo) {+ lastSpokenNo = log.no+ speech.speak(log.content)+ return+ }+ }+ }+}+function updateProgress(progress) {if (!progress) progress = "Waiting for input"@@ -453,6 +476,7 @@ const setContext = function (id) {context = idlastLogGuid = ""lastLogVersion = 0+ lastSpokenNo = 0const chatsAD = Alpine.$data(chatsSection);chatsAD.selected = id}@@ -488,6 +512,12 @@ window.toggleDarkMode = function (isDark) {localStorage.setItem('darkMode', isDark);};+window.toggleSpeech = function (isOn) {+ console.log("Speech:", isOn);+ localStorage.setItem('speech', isOn);+ if(!isOn) speech.stop()+};+// Modify this partdocument.addEventListener('DOMContentLoaded', () => {const isDarkMode = localStorage.getItem('darkMode') !== 'false';@@ -576,20 +606,20 @@ window.saveChat = async function () {function downloadFile(filename, content) {// Create a Blob with the content to saveconst blob = new Blob([content], { type: 'application/json' });-+// Create a link elementconst link = document.createElement('a');-+// Create a URL for the Blobconst url = URL.createObjectURL(blob);link.href = url;-+// Set the file name for downloadlink.download = filename;-+// Programmatically click the link to trigger the downloadlink.click();-+// Clean up by revoking the object URLsetTimeout(() => {URL.revokeObjectURL(url);diff --git a/webui/messages.js b/webui/messages.jsindex 8e45a1137..a2639d8e5 100644--- a/webui/messages.js+++ b/webui/messages.js@@ -62,10 +62,10 @@ export function _drawMessage(messageContainer, heading, content, temp, followUp,if (window.renderMathInElement) {renderMathInElement(spanElement, {delimiters: [- {left: "$", right: "$", display: true},- {left: "\\$", right: "\\$", display: true},- {left: "$", right: "$", display: false},- {left: "\\$", right: "\\$", display: false}+ { left: "$", right: "$", display: true },+ { left: "\\$", right: "\\$", display: true },+ { left: "$", right: "$", display: false },+ { left: "\\$", right: "\\$", display: false }],throwOnError: false // Prevent KaTeX from throwing errors});@@ -115,7 +115,7 @@ export function drawMessageUser(messageContainer, id, type, heading, content, teif (hasAttachments) {const attachmentsContainer = document.createElement('div');attachmentsContainer.classList.add('attachments-container');-+// Add "Attachments" heading if there's text contentif (hasContent) {const attachmentsHeading = document.createElement('h4');@@ -204,10 +204,10 @@ function drawKvps(container, kvps) {if (window.renderMathInElement) {renderMathInElement(span, {delimiters: [- {left: "$$", right: "$$", display: true},- {left: "\$$", right: "\$$", display: true},- {left: "$", right: "$", display: false},- {left: "\$$", right: "\$$", display: false}+ { left: "$$", right: "$$", display: true },+ { left: "\$$", right: "\$$", display: true },+ { left: "$", right: "$", display: false },+ { left: "\$$", right: "\$$", display: false }],throwOnError: false // Prevent KaTeX from throwing errors});diff --git a/webui/speech copy.js b/webui/speech copy.jsdeleted file mode 100644index eb2050075..000000000--- a/webui/speech copy.js+++ /dev/null@@ -1,337 +0,0 @@-import { pipeline, read_audio } from './[email protected]';-import { updateChatInput, sendMessage } from './index.js';--const microphoneButton = document.getElementById('microphone-button');-let microphoneInput = null;-let isProcessingClick = false;--class MicrophoneInput {- /**- * Voice Input Handler with Whisper Transcription- *- * Whisper Model Size Configuration:- * - 'tiny': Smallest model, fastest, lowest accuracy (~32MB)- * - Best for: Quick prototyping, low-resource environments- * - Pros: Very fast, low memory usage- * - Cons: Lowest transcription accuracy- *- * - 'base': Small model, good balance of speed and accuracy (~74MB)- * - Best for: General-purpose voice input- * - Pros: Reasonable accuracy, moderate resource usage- * - Cons: Less accurate than larger models- *- * - 'small': Medium-sized model, better accuracy (~244MB)- * - Best for: More precise transcription needs- * - Pros: Improved accuracy over base model- * - Cons: Slower, more memory-intensive- *- * - 'medium': Large model with high accuracy (~769MB)- * - Best for: Professional transcription, multi-language support- * - Pros: Very high accuracy- * - Cons: Significant computational resources required- *- * - 'large': Largest model, highest accuracy (~1.5GB)- * - Best for: Professional, multi-language transcription- * - Pros: Highest possible accuracy- * - Cons: Slowest, most resource-intensive- *- * Recommended Default: 'base' for most web applications- */- constructor(updateCallback, options = {}) {- this.mediaRecorder = null;- this.audioChunks = [];- this.isRecording = false;- this.updateCallback = updateCallback;- this.isFinalizing = false;- this.messageSent = false; // move messageSent into class-- // New properties for silence detection- this.audioContext = null;- this.mediaStreamSource = null;- this.analyserNode = null;- this.silenceTimer = null;- this.silenceThreshold = options.silenceThreshold || 0.01; // Adjust as needed- this.silenceDuration = options.silenceDuration || 2000; // Duration in milliseconds-- this.options = {- modelSize: 'tiny',- language: 'en',- chunkDuration: 3000,- ...options- };- }-- async initialize() {- try {-- this.transcriber = await pipeline(`automatic-speech-recognition`, `Xenova/whisper-${this.options.modelSize}.${this.options.language}`);-- const stream = await navigator.mediaDevices.getUserMedia({- audio: {- echoCancellation: true,- noiseSuppression: true,- channelCount: 1- }- });-- // Configure MediaRecorder- this.mediaRecorder = new MediaRecorder(stream);-- // Handle audio data availability- this.mediaRecorder.ondataavailable = async (event) => {- if (event.data.size > 0) {- this.audioChunks.push(event.data);- // await this.processAudioChunk(event.data);- }- };-- // Handle recording stop- this.mediaRecorder.onstop = async () => {- await this.finalizeRecording();- };-- // Set up AudioContext and AnalyserNode for silence detection- this.audioContext = new (window.AudioContext || window.webkitAudioContext)();- this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);- this.analyserNode = this.audioContext.createAnalyser();- this.analyserNode.minDecibels = -90;- this.analyserNode.maxDecibels = -10;- this.analyserNode.smoothingTimeConstant = 0.85;-- this.mediaStreamSource.connect(this.analyserNode);- } catch (error) {- console.error('Microphone initialization error:', error);- toast('Failed to access microphone. Please check permissions.', 'error');- }- }-- startSilenceDetection() {- const dataArray = new Uint8Array(this.analyserNode.fftSize);- const checkSilence = () => {- this.analyserNode.getByteTimeDomainData(dataArray);-- let sum = 0;- for (let i = 0; i < dataArray.length; i++) {- const amplitude = (dataArray[i] - 128) / 128;- sum += amplitude * amplitude;- }- const rms = Math.sqrt(sum / dataArray.length);-- if (rms < this.silenceThreshold) {- if (!this.silenceTimer) {- this.silenceTimer = setTimeout(() => {- if (this.isRecording) {- console.log('Silence detected. Stopping recording.');- this.stopRecording();- microphoneButton.classList.remove('recording');- microphoneButton.classList.remove('mic-pulse');- }- }, this.silenceDuration);- }- } else {- if (this.silenceTimer) {- clearTimeout(this.silenceTimer);- this.silenceTimer = null;- }- }-- if (this.isRecording) {- requestAnimationFrame(checkSilence);- }- };-- if (this.isRecording) {- requestAnimationFrame(checkSilence);- }- }-- startRecording() {- if (this.mediaRecorder && this.audioContext) {- this.isRecording = true;- this.audioChunks = [];- this.messageSent = false;- this.mediaRecorder.start(this.options.chunkDuration);- this.audioContext.resume();- this.startSilenceDetection();- }- }-- stopRecording() {- if (this.mediaRecorder && this.isRecording) {- this.isRecording = false;- if (!this.isFinalizing) {- this.isFinalizing = true;- this.mediaRecorder.stop();- this.audioContext.suspend();- if (this.silenceTimer) {- clearTimeout(this.silenceTimer);- this.silenceTimer = null;- }- }- }- }--- async finalizeRecording() {- if (this.isFinalizing) {- this.isFinalizing = false;-- if (this.audioChunks.length > 0) {-- const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });- const audioUrl = URL.createObjectURL(audioBlob);- const samplingRate = 16000; // Adjust as needed for the model- const audioData = await read_audio(audioUrl, samplingRate);- URL.revokeObjectURL(audioUrl);-- // Transcribe the audio- const result = await this.transcriber(audioData);-- if (result.text) {- console.log('Final transcription received:', result.text);- await this.updateCallback(result.text, true);- } else {- console.warn('Final transcription returned empty text.');- }--- // Release the object URL after use-- // const audioBlob = new Blob(this.audioChunks, { type: 'audio/webm' });- // this.audioChunks = []; // Clear for next recording-- // const reader = new FileReader();- // reader.onloadend = async () => {- // const base64Data = reader.result.split(',')[1];-- // try {- // const response = await fetch('/transcribe', {- // method: 'POST',- // headers: {- // 'Content-Type': 'application/json'- // },- // body: JSON.stringify({- // audio_data: base64Data,- // model_size: this.options.modelSize,- // language: this.options.language,- // is_final: true- // })- // });-- // const result = await response.json();-- // if (result.text) {- // console.log('Final transcription received:', result.text);- // await this.updateCallback(result.text, true);- // } else {- // console.warn('Final transcription returned empty text.');- // }- // } catch (transcribeError) {- // console.error('Final transcription error:', transcribeError);- // toast('Final transcription failed.', 'error');- // } finally {- // // Reset the microphone button state- // microphoneButton.classList.remove('recording');- // microphoneButton.classList.remove('mic-pulse');- // microphoneButton.style.backgroundColor = '';- // }- // };- // reader.readAsDataURL(audioBlob);- }- }- }-}--export default MicrophoneInput;--async function initializeMicrophoneInput() {- console.log('Initializing microphone input');-- microphoneInput = new MicrophoneInput(- async (text, isFinal) => {- if (isFinal) {- console.log('Final transcription callback received:', text);- updateChatInput(text)- // chatInput.value = text;- // adjustTextareaHeight();-- if (!microphoneInput.messageSent) {- microphoneInput.messageSent = true;- console.log('Sending message');- await sendMessage();- }- }- },- {- modelSize: 'tiny',- language: 'en',- silenceThreshold: 0.07, // Adjust as needed- silenceDuration: 2000, // Adjust as needed- onError: (error) => {- console.error('Microphone input error:', error);- toast('Microphone error: ' + error.message, 'error');- // Reset recording state- if (microphoneButton.classList.contains('recording')) {- microphoneButton.classList.remove('recording');- }- }- }- );-- await microphoneInput.initialize();-}---function toggleRecording() {- console.log('toggleRecording called, isRecording:', microphoneInput.isRecording);-- if (microphoneInput.isRecording) {- microphoneInput.stopRecording();- microphoneButton.classList.remove('recording');- // Add pulsing animation class- microphoneButton.classList.remove('mic-pulse');- } else {- microphoneInput.startRecording();- microphoneButton.classList.add('recording');- // Add pulsing animation class- microphoneButton.classList.add('mic-pulse');- }-- // Add visual feedback- microphoneButton.style.backgroundColor = microphoneInput.isRecording ? '#ff4444' : '';- console.log('New recording state:', microphoneInput.isRecording);-}--// Some error handling for microphone input-async function requestMicrophonePermission() {- try {- await navigator.mediaDevices.getUserMedia({ audio: true });- return true;- } catch (err) {- console.error('Error accessing microphone:', err);- toast('Microphone access denied. Please enable microphone access in your browser settings.', 'error');- return false;- }-}-// microphoneButton click event listener modifier-microphoneButton.addEventListener('click', async () => {- console.log('Microphone button clicked');- if (isProcessingClick) {- console.log('Click already being processed, ignoring');- return;- }- isProcessingClick = true;-- const hasPermission = await requestMicrophonePermission();- if (!hasPermission) return;-- if (!microphoneInput) {- await initializeMicrophoneInput();- }-- await toggleRecording();-- setTimeout(() => {- isProcessingClick = false;- }, 300); // Add a 300ms delay before allowing another click-});diff --git a/webui/speech.js b/webui/speech.jsindex f7dd0055d..69c545fb7 100644--- a/webui/speech.js+++ b/webui/speech.js@@ -217,7 +217,8 @@ class MicrophoneInput {this.silenceStartTime = null;if (this.status === Status.LISTENING || this.status === Status.WAITING) {- this.status = Status.RECORDING;+ if (!speech.isSpeaking()) // TODO? a better way to ignore agent's voice?+ this.status = Status.RECORDING;}} else if (this.status === Status.RECORDING) {if (!this.silenceStartTime) {@@ -347,3 +348,44 @@ async function requestMicrophonePermission() {return false;}}+++class Speech {+ constructor() {+ this.synth = window.speechSynthesis;+ this.utterance = null;+ }++ stripEmojis(str) {+ return str+ .replace(/([\u2700-\u27BF]|[\uE000-\uF8FF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDFFF]|[\u2011-\u26FF]|\uD83E[\uDD10-\uDDFF])/g, '')+ .replace(/\s+/g, ' ')+ .trim();+ }++ speak(text) {+ console.log('Speaking:', text);+ // Stop any current utterance+ this.stop();++ // Remove emojis and create a new utterance+ text = this.stripEmojis(text);+ this.utterance = new SpeechSynthesisUtterance(text);++ // Speak the new utterance+ this.synth.speak(this.utterance);+ }++ stop() {+ if (this.isSpeaking()) {+ this.synth.cancel();+ }+ }++ isSpeaking() {+ return this.synth?.speaking || false;+ }+}++export const speech = new Speech();+window.speech = speech\ No newline at end of filediff --git a/webui/test.html b/webui/test.htmldeleted file mode 100644index 7856867fb..000000000--- a/webui/test.html+++ /dev/null@@ -1,82 +0,0 @@-<!DOCTYPE html>-<html lang="en">-<head>- <meta charset="UTF-8">- <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">- <title>Agent Zero</title>-- <script type="module">- import { pipeline, read_audio } from './[email protected]';-- let transcriber;- let mediaRecorder;- let audioChunks = [];- let isRecording = false;-- // Initialize the transcriber- async function initTranscriber() {- transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');- }-- // Toggle recording state- async function toggleRecording() {- if (isRecording) {- stopRecording();- } else {- startRecording();- }- }-- // Start recording from the microphone- async function startRecording() {- isRecording = true;- audioChunks = [];- document.getElementById("micButton").innerText = "Stop Recording";-- const stream = await navigator.mediaDevices.getUserMedia({ audio: true });- mediaRecorder = new MediaRecorder(stream);-- mediaRecorder.ondataavailable = event => {- audioChunks.push(event.data);- };-- mediaRecorder.onstop = async () => {- const audioBlob = new Blob(audioChunks, { type: 'audio/wav' });- const audioUrl = URL.createObjectURL(audioBlob);-- // Use read_audio to fetch and process the audio from the URL- const samplingRate = 16000; // Adjust as needed for the model- const audioData = await read_audio(audioUrl, samplingRate);-- // Transcribe the audio- const output = await transcriber(audioData);- document.getElementById("transcript").innerText = output.text;-- // Release the object URL after use- URL.revokeObjectURL(audioUrl);- };-- mediaRecorder.start();- }-- // Stop recording- function stopRecording() {- isRecording = false;- document.getElementById("micButton").innerText = "Start Recording";- mediaRecorder.stop();- }-- // Expose functions to the global scope- window.toggleRecording = toggleRecording;-- // Initialize the transcriber on page load- window.onload = initTranscriber;- </script>-</head>--<body>- <h1>Agent Zero Speech Transcription</h1>- <button id="micButton" onclick="toggleRecording()">Start Recording</button>- <p id="transcript">Transcript will appear here...</p>-</body>-</html>diff --git a/webui/test2.html b/webui/test2.htmldeleted file mode 100644index 02e431458..000000000--- a/webui/test2.html+++ /dev/null@@ -1,122 +0,0 @@-<!DOCTYPE html>-<html lang="en">-<head>- <meta charset="UTF-8">- <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">- <title>Agent Zero</title>-- <script type="module">- import { pipeline, read_audio } from './[email protected]';-- let transcriber;- let mediaRecorder;- let isRecording = false;- let audioChunks = [];- let chunks_to_process = [{ tokens: [], finalized: false }];-- async function initTranscriber() {- try {- transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');- } catch (error) {- console.error("Failed to initialize transcriber:", error);- }- }-- async function toggleRecording() {- if (isRecording) {- stopRecording();- } else {- startRecording();- }- }-- async function startRecording() {- isRecording = true;- document.getElementById("micButton").innerText = "Stop Recording";-- const stream = await navigator.mediaDevices.getUserMedia({ audio: true });- mediaRecorder = new MediaRecorder(stream, { mimeType: 'audio/webm;codecs=opus' });-- // Handle data as it’s available- mediaRecorder.ondataavailable = (event) => {- audioChunks.push(event.data);-- // Process chunks every 3 seconds- if (audioChunks.length >= 3) {- processAudioChunks();- }- };-- // Capture small chunks every second- mediaRecorder.start(1000);- }-- async function processAudioChunks() {- const audioBlob = new Blob(audioChunks, { type: 'audio/webm' });- const audioUrl = URL.createObjectURL(audioBlob);-- try {- const audioData = await read_audio(audioUrl, 16000);-- if (transcriber) {- await transcriber(audioData, {- callback_function: mergeChunksCallback,- chunk_callback: processChunkCallback,- return_timestamps: true,- force_full_sequences: false,- });- } else {- console.warn("Transcriber is not ready yet.");- }- } catch (error) {- console.error("Error during transcription:", error);- }-- // Cleanup and reset buffer- URL.revokeObjectURL(audioUrl);- audioChunks = [];- }-- function processChunkCallback(chunk) {- let lastChunk = chunks_to_process[chunks_to_process.length - 1];- Object.assign(lastChunk, chunk);- lastChunk.finalized = true;-- if (!chunk.is_last) {- chunks_to_process.push({ tokens: [], finalized: false });- }- }-- function mergeChunksCallback(item) {- let lastChunk = chunks_to_process[chunks_to_process.length - 1];- lastChunk.tokens = [...item[0].output_token_ids];-- // Merge text chunks and update transcript- const mergedText = transcriber.tokenizer._decode_asr(chunks_to_process, {- time_precision: 1.0 / transcriber.processor.feature_extractor.config.chunk_length,- return_timestamps: true,- force_full_sequences: false,- });-- document.getElementById("transcript").innerText += mergedText + ' ';- }-- function stopRecording() {- isRecording = false;- document.getElementById("micButton").innerText = "Start Recording";- if (mediaRecorder && mediaRecorder.state !== 'inactive') {- mediaRecorder.stop();- }- }-- window.toggleRecording = toggleRecording;- window.onload = initTranscriber;- </script>-</head>--<body>- <h1>Agent Zero Speech Transcription</h1>- <button id="micButton" onclick="toggleRecording()">Start Recording</button>- <p id="transcript">Transcript will appear here...</p>-</body>-</html>