Skip to content

Agent Zero AI framework

TTS prototype

frdel92d94b4d86ef
diff --git a/webui/index.css b/webui/index.css
index fde8a6c4e..cb3848478 100644
--- a/webui/index.css
+++ b/webui/index.css
@@ -446,6 +446,8 @@ pre {
padding-bottom: 0;*/
padding: var(--spacing-sm) var(--spacing-md);
padding-bottom: 0;
+ display: flex;
+ justify-content: space-between;
}
#progress-bar-h {
diff --git a/webui/index.html b/webui/index.html
index 309670d60..131467b6e 100644
--- a/webui/index.html
+++ b/webui/index.html
@@ -9,7 +9,7 @@
<link rel="stylesheet" href="toast.css">
<link rel="stylesheet" href="settings.css">
<link rel="stylesheet" href="speech.css">
-
+
<script>
window.safeCall = function (name, ...args) {
if (window[name]) window[name](...args)
@@ -21,7 +21,8 @@
<!-- KaTeX javascript -->
<script defer src="https://cdn.jsdelivr.net/npm/[email protected]/dist/katex.min.js" crossorigin="anonymous"></script>
- <script defer src="https://cdn.jsdelivr.net/npm/[email protected]/dist/contrib/auto-render.min.js" crossorigin="anonymous"></script>
+ <script defer src="https://cdn.jsdelivr.net/npm/[email protected]/dist/contrib/auto-render.min.js"
+ crossorigin="anonymous"></script>
<script defer src="https://cdn.jsdelivr.net/npm/@alpinejs/[email protected]/dist/cdn.min.js"></script>
<script defer src="https://cdn.jsdelivr.net/npm/[email protected]/dist/cdn.min.js"></script>
@@ -39,23 +40,24 @@
<button id="toggle-sidebar" class="toggle-sidebar-button" aria-label="Toggle Sidebar" aria-expanded="false">
<span aria-hidden="true">
<!-- Hamburger Icon -->
- <svg id="sidebar-hamburger-svg" xmlns="http://www.w3.org/2000/svg" width="22" height="22" viewBox="0 0 24 24" fill="CurrentColor">
- <path d="M3 13h18v-2H3v2zm0 4h18v-2H3v2zm0-8h18V7H3v2z"></path>
+ <svg id="sidebar-hamburger-svg" xmlns="http://www.w3.org/2000/svg" width="22" height="22"
+ viewBox="0 0 24 24" fill="CurrentColor">
+ <path d="M3 13h18v-2H3v2zm0 4h18v-2H3v2zm0-8h18V7H3v2z"></path>
</svg>
</span>
</button>
-
- <div id="logo-container">
+
+ <div id="logo-container">
<a href="https://github.com/frdel/agent-zero" target="_blank" rel="noopener noreferrer">
<img src="splash.jpg" alt="a0" width="22" height="22">
</a>
+ </div>
</div>
- </div>
-
- <div id="left-panel" class="panel">
- <!--Sidebar upper elements-->
- <div class="left-panel-top">
- <div class="config-section" id="status-section" x-data="{ connected: true }">
+
+ <div id="left-panel" class="panel">
+ <!--Sidebar upper elements-->
+ <div class="left-panel-top">
+ <div class="config-section" id="status-section" x-data="{ connected: true }">
<h3>Status</h3>
<h4 class="connected" x-show="connected">&#10004; Connected</h4>
<h4 class="disconnected" x-show="!connected">&#10008; Disconnected</h4>
@@ -72,85 +74,95 @@
</div>
<!-- Chats List -->
<div class="config-section" id="chats-section" x-data="{ contexts: [], selected: '' }"
- x-show="contexts.length > 0">
- <h3>Chats</h3>
- <div class="chats-list-container">
- <ul class="config-list">
- <template x-for="context in contexts">
- <li>
- <span :class="{'chat-list-button': true, 'font-bold': context.id === selected}"
- @click="selected = context.id; selectChat(context.id)">
- Chat #<span x-text="context.no"></span>
- </span>
- <button class="edit-button" @click="killChat(context.id)">X</button>
- </li>
- </template>
- </ul>
+ x-show="contexts.length > 0">
+ <h3>Chats</h3>
+ <div class="chats-list-container">
+ <ul class="config-list">
+ <template x-for="context in contexts">
+ <li>
+ <span :class="{'chat-list-button': true, 'font-bold': context.id === selected}"
+ @click="selected = context.id; selectChat(context.id)">
+ Chat #<span x-text="context.no"></span>
+ </span>
+ <button class="edit-button" @click="killChat(context.id)">X</button>
+ </li>
+ </template>
+ </ul>
+ </div>
</div>
</div>
- </div>
- <!--Sidebar lower elements-->
- <div class="left-panel-bottom">
- <!-- Preferences -->
- <div class="pref-section" x-data="{ prefOpen: true }">
- <span><h3 class="pref-header" @click="prefOpen = !prefOpen">
- Preferences
- <svg class="arrow-icon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"
- fill="none" stroke="currentColor" stroke-width="2" width="16" height="16"
- :class="{'rotated': !prefOpen}">
- <path d="M8 4l8 8-8 8"/>
- </svg>
- </h3>
- <ul class="config-list" x-show="prefOpen" x-collapse x-transition>
- <!-- Preferences Items -->
- <li x-data="{ autoScroll: true }">
- <span>Autoscroll</span>
- <label class="switch">
- <input id="auto-scroll-switch" type="checkbox" x-model="autoScroll"
- x-effect="window.safeCall('toggleAutoScroll',autoScroll)">
- <span class="slider"></span>
- </label>
- </li>
- <li x-data="{ darkMode: localStorage.getItem('darkMode') != 'false' }"
- x-init="$watch('darkMode', val => toggleDarkMode(val))">
- <span class="switch-label">Dark mode</span>
- <label class="switch">
- <input type="checkbox" x-model="darkMode">
- <span class="slider"></span>
- </label>
- </li>
- <li x-data="{ showThoughts: true }">
- <span>Show thoughts</span>
- <label class="switch">
- <input type="checkbox" x-model="showThoughts"
- x-effect="window.safeCall('toggleThoughts',showThoughts)">
- <span class="slider"></span>
- </label>
- </li>
- <li x-data="{ showJson: false }">
- <span>Show JSON</span>
- <label class="switch">
- <input type="checkbox" x-model="showJson" x-effect="window.safeCall('toggleJson',showJson)">
- <span class="slider"></span>
- </label>
- </li>
- <li x-data="{ showUtils: false }">
- <span>Show utility messages</span>
- <label class="switch">
- <input type="checkbox" x-model="showUtils"
- x-effect="window.safeCall('toggleUtils',showUtils)">
- <span class="slider"></span>
- </label>
- </li>
- </ul>
- </span>
- </div>
- <!-- Version Info -->
- <div class="version-info">
- <span id="a0version">Agent Zero 0.7.2<br>built on 2024-11-04</span>
+ <!--Sidebar lower elements-->
+ <div class="left-panel-bottom">
+ <!-- Preferences -->
+ <div class="pref-section" x-data="{ prefOpen: true }">
+ <span>
+ <h3 class="pref-header" @click="prefOpen = !prefOpen">
+ Preferences
+ <svg class="arrow-icon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
+ stroke="currentColor" stroke-width="2" width="16" height="16"
+ :class="{'rotated': !prefOpen}">
+ <path d="M8 4l8 8-8 8" />
+ </svg>
+ </h3>
+ <ul class="config-list" x-show="prefOpen" x-collapse x-transition>
+ <!-- Preferences Items -->
+ <li x-data="{ autoScroll: true }">
+ <span>Autoscroll</span>
+ <label class="switch">
+ <input id="auto-scroll-switch" type="checkbox" x-model="autoScroll"
+ x-effect="window.safeCall('toggleAutoScroll',autoScroll)">
+ <span class="slider"></span>
+ </label>
+ </li>
+ <li x-data="{ darkMode: localStorage.getItem('darkMode') != 'false' }"
+ x-init="$watch('darkMode', val => toggleDarkMode(val))">
+ <span class="switch-label">Dark mode</span>
+ <label class="switch">
+ <input type="checkbox" x-model="darkMode">
+ <span class="slider"></span>
+ </label>
+ </li>
+ <li x-data="{ speech: localStorage.getItem('speech') == 'true' }"
+ x-init="$watch('speech', val => toggleSpeech(val))">
+ <span class="switch-label">Speech</span>
+ <label class="switch">
+ <input type="checkbox" x-model="speech">
+ <span class="slider"></span>
+ </label>
+ </li>
+ <li x-data="{ showThoughts: true }">
+ <span>Show thoughts</span>
+ <label class="switch">
+ <input type="checkbox" x-model="showThoughts"
+ x-effect="window.safeCall('toggleThoughts',showThoughts)">
+ <span class="slider"></span>
+ </label>
+ </li>
+ <li x-data="{ showJson: false }">
+ <span>Show JSON</span>
+ <label class="switch">
+ <input type="checkbox" x-model="showJson"
+ x-effect="window.safeCall('toggleJson',showJson)">
+ <span class="slider"></span>
+ </label>
+ </li>
+ <li x-data="{ showUtils: false }">
+ <span>Show utility messages</span>
+ <label class="switch">
+ <input type="checkbox" x-model="showUtils"
+ x-effect="window.safeCall('toggleUtils',showUtils)">
+ <span class="slider"></span>
+ </label>
+ </li>
+ </ul>
+ </span>
+ </div>
+ <!-- Version Info -->
+ <div class="version-info">
+ <span id="a0version">Agent Zero 0.7.2<br>built on 2024-11-04</span>
+ </div>
</div>
</div>
- </div>
<div id="right-panel" class="panel">
<!--Chat-->
<div id="time-date"></div>
@@ -161,8 +173,16 @@
<button class="toast__copy">Copy</button>
<button class="toast__close">Close</button>
</div>
- <div id="progress-bar-box">
- <h4 id="progress-bar-h"><span id="progress-bar-i">|></span><span id="progress-bar"></span></h4>
+ <div id="progress-bar-box" x-data="{ isSpeaking: false }" x-init="
+ // Watch the speech synthesis status to update isSpeaking
+ setInterval(() => isSpeaking = window.speech.isSpeaking(), 100)
+ ">
+ <h4 id="progress-bar-h">
+ <span id="progress-bar-i">|></span><span id="progress-bar"></span>
+ </h4>
+ <h4 id="progress-bar-stop-speech" x-show="isSpeaking">
+ <span id="stop-speech" @click="window.speech.stop()" style="cursor: pointer">Stop Speech</span>
+ </h4>
</div>
<div id="input-section" x-data="{
paused: false,
@@ -197,8 +217,8 @@
<template x-for="(image, index) in attachments" :key="index">
<div class="preview-item">
<img :src="image.url" alt="Preview">
- <button @click="attachments.splice(index, 1); hasAttachments = attachments.length > 0"
- class="remove-image">&times;</button>
+ <button @click="attachments.splice(index, 1); hasAttachments = attachments.length > 0"
+ class="remove-image">&times;</button>
</div>
</template>
</div>
@@ -207,65 +227,80 @@
<div class="input-row">
<!-- Attachment icon with tooltip -->
<div class="attachment-wrapper" x-data="{ showTooltip: false }">
- <label for="file-input" class="attachment-icon"
- @mouseover="showTooltip = true"
- @mouseleave="showTooltip = false">
- <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="currentColor">
- <path d="M16.5 6v11.5c0 2.21-1.79 4-4 4s-4-1.79-4-4V5c0-1.38 1.12-2.5 2.5-2.5s2.5 1.12 2.5 2.5v10.5c0 .55-.45 1-1 1s-1-.45-1-1V6H10v9.5c0 1.38 1.12 2.5 2.5 2.5s2.5-1.12 2.5-2.5V5c0-2.21-1.79-4-4-4S7 2.79 7 5v12.5c0 3.04 2.46 5.5 5.5 5.5s5.5-2.46 5.5-5.5V6h-1.5z"/>
+ <label for="file-input" class="attachment-icon" @mouseover="showTooltip = true"
+ @mouseleave="showTooltip = false">
+ <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24"
+ fill="currentColor">
+ <path
+ d="M16.5 6v11.5c0 2.21-1.79 4-4 4s-4-1.79-4-4V5c0-1.38 1.12-2.5 2.5-2.5s2.5 1.12 2.5 2.5v10.5c0 .55-.45 1-1 1s-1-.45-1-1V6H10v9.5c0 1.38 1.12 2.5 2.5 2.5s2.5-1.12 2.5-2.5V5c0-2.21-1.79-4-4-4S7 2.79 7 5v12.5c0 3.04 2.46 5.5 5.5 5.5s5.5-2.46 5.5-5.5V6h-1.5z" />
</svg>
</label>
- <input type="file" id="file-input"
- accept="png, jpg, jpeg, txt, pdf, csv, html, json, md"
- multiple
- style="display: none"
- @change="handleFileUpload($event)">
+ <input type="file" id="file-input" accept="png, jpg, jpeg, txt, pdf, csv, html, json, md"
+ multiple style="display: none" @change="handleFileUpload($event)">
<div x-show="showTooltip" class="tooltip">
Limit: 4 attachments per message
</div>
</div>
-
+
<!-- Text input -->
<textarea id="chat-input" placeholder="Type your message here..." rows="1"></textarea>
-
+
<div id="chat-buttons-wrapper">
- <!-- Send button -->
- <button class="chat-button" id="send-button" aria-label="Send message">
- <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
- <path d="M25 20 L75 50 L25 80" fill="none" stroke="currentColor" stroke-width="15"></path>
- </svg>
- </button>
-
- <!-- Microphone button -->
- <button class="chat-button mic-inactive" id="microphone-button" aria-label="Start/Stop recording">
- <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 18" fill="currentColor" width="24" height="24">
- <path d="m8,12c1.66,0,3-1.34,3-3V3c0-1.66-1.34-3-3-3s-3,1.34-3,3v6c0,1.66,1.34,3,3,3Zm-1,1.9c-2.7-.4-4.8-2.6-5-5.4H0c.2,3.8,3.1,6.9,7,7.5v2h2v-2c3.9-.6,6.8-3.7,7-7.5h-2c-.2,2.8-2.3,5-5,5.4h-2Z"/>
- </svg>
+ <!-- Send button -->
+ <button class="chat-button" id="send-button" aria-label="Send message">
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
+ <path d="M25 20 L75 50 L25 80" fill="none" stroke="currentColor" stroke-width="15">
+ </path>
+ </svg>
+ </button>
- </button>
+ <!-- Microphone button -->
+ <button class="chat-button mic-inactive" id="microphone-button"
+ aria-label="Start/Stop recording">
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 18" fill="currentColor" width="24"
+ height="24">
+ <path
+ d="m8,12c1.66,0,3-1.34,3-3V3c0-1.66-1.34-3-3-3s-3,1.34-3,3v6c0,1.66,1.34,3,3,3Zm-1,1.9c-2.7-.4-4.8-2.6-5-5.4H0c.2,3.8,3.1,6.9,7,7.5v2h2v-2c3.9-.6,6.8-3.7,7-7.5h-2c-.2,2.8-2.3,5-5,5.4h-2Z" />
+ </svg>
+
+ </button>
</div>
</div>
-
+
<!-- Bottom row with text buttons -->
<div class="text-buttons-row">
- <button class="text-button" @click="loadKnowledge()"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M3 16.5v2.25A2.25 2.25 0 0 0 5.25 21h13.5A2.25 2.25 0 0 0 21 18.75V16.5m-13.5-9L12 3m0 0 4.5 4.5M12 3v13.5"></path></svg>Import knowledge</button>
+ <button class="text-button" @click="loadKnowledge()"><svg xmlns="http://www.w3.org/2000/svg"
+ fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor">
+ <path stroke-linecap="round" stroke-linejoin="round"
+ d="M3 16.5v2.25A2.25 2.25 0 0 0 5.25 21h13.5A2.25 2.25 0 0 0 21 18.75V16.5m-13.5-9L12 3m0 0 4.5 4.5M12 3v13.5">
+ </path>
+ </svg>Import knowledge</button>
<button class="text-button" @click="workDirModalProxy.openModal()">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 123.37 92.59">
- <path d="m5.72,11.5l-3.93,8.73h119.77s-3.96-8.73-3.96-8.73h-60.03c-1.59,0-2.88-1.29-2.88-2.88V1.75H13.72v6.87c0,1.59-1.29,2.88-2.88,2.88h-5.12Z" fill="none" stroke="currentColor" stroke-linejoin="round" stroke-width="5"></path>
- <path d="m6.38,20.23H1.75l7.03,67.03c.11,1.07.55,2.02,1.2,2.69.55.55,1.28.89,2.11.89h97.1c.82,0,1.51-.33,2.05-.87.68-.68,1.13-1.67,1.28-2.79l9.1-66.94H6.38Z" fill="none" stroke="currentColor" stroke-linejoin="round" stroke-width="5"></path>
- </svg>work_dir Browser</button>
-
+ <path
+ d="m5.72,11.5l-3.93,8.73h119.77s-3.96-8.73-3.96-8.73h-60.03c-1.59,0-2.88-1.29-2.88-2.88V1.75H13.72v6.87c0,1.59-1.29,2.88-2.88,2.88h-5.12Z"
+ fill="none" stroke="currentColor" stroke-linejoin="round" stroke-width="5"></path>
+ <path
+ d="m6.38,20.23H1.75l7.03,67.03c.11,1.07.55,2.02,1.2,2.69.55.55,1.28.89,2.11.89h97.1c.82,0,1.51-.33,2.05-.87.68-.68,1.13-1.67,1.28-2.79l9.1-66.94H6.38Z"
+ fill="none" stroke="currentColor" stroke-linejoin="round" stroke-width="5"></path>
+ </svg>work_dir Browser</button>
+
<button class="text-button" @click="pauseAgent(!paused)">
- <!-- Dynamic path that switches between pause and play icons -->
- <template x-if="!paused">
- <svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" width="14" height="14">
- <path d="M6 19h4V5H6v14zm8-14v14h4V5h-4z"></path></svg>
- </template>
- <template x-if="paused">
- <svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" width="14" height="14">
- <path d="M8 5v14l11-7z"></path></svg>
- </template>
+ <!-- Dynamic path that switches between pause and play icons -->
+ <template x-if="!paused">
+ <svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" viewBox="0 0 24 24"
+ stroke-width="1.5" stroke="currentColor" width="14" height="14">
+ <path d="M6 19h4V5H6v14zm8-14v14h4V5h-4z"></path>
+ </svg>
+ </template>
+ <template x-if="paused">
+ <svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" viewBox="0 0 24 24"
+ stroke-width="1.5" stroke="currentColor" width="14" height="14">
+ <path d="M8 5v14l11-7z"></path>
+ </svg>
+ </template>
</svg>
<span x-text="paused ? 'Resume Agent' : 'Pause Agent'"></span>
</button>
@@ -275,8 +310,7 @@
</div>
<div id="settingsModal" x-data="settingsModalProxy">
<template x-teleport="body">
- <div x-show="isOpen" class="modal-overlay" @click.self="handleCancel()"
- x-transition>
+ <div x-show="isOpen" class="modal-overlay" @click.self="handleCancel()" x-transition>
<div class="modal-container">
<div class="modal-header">
<h2 x-text="settings.title"></h2>
@@ -369,10 +403,10 @@
<div class="modal-footer">
<div id="buttons-container">
- <template x-for="button in settings.buttons" :key="button.id">
- <button :class="button.classes" @click="handleButton(button.id)"
- x-text="button.title"></button>
- </template>
+ <template x-for="button in settings.buttons" :key="button.id">
+ <button :class="button.classes" @click="handleButton(button.id)"
+ x-text="button.title"></button>
+ </template>
</div>
</div>
</div>
diff --git a/webui/index.js b/webui/index.js
index 2b87b9875..e0e2fe15b 100644
--- a/webui/index.js
+++ b/webui/index.js
@@ -1,4 +1,5 @@
import * as msgs from "./messages.js"
+import { speech } from "./speech.js";
const leftPanel = document.getElementById('left-panel');
const rightPanel = document.getElementById('right-panel');
@@ -59,7 +60,7 @@ function setupSidebarToggle() {
// Make sure to call this function
document.addEventListener('DOMContentLoaded', setupSidebarToggle);
- // index.js
+// index.js
export async function sendMessage() {
try {
const message = chatInput.value.trim();
@@ -77,7 +78,7 @@ export async function sendMessage() {
...attachment,
url: URL.createObjectURL(attachment.file)
}));
-
+
// Only render if there's text content or it's an image-only message
setMessage(messageId, 'user', '', message, false, {
attachments: attachmentsWithUrls
@@ -98,10 +99,10 @@ export async function sendMessage() {
});
} else {
// For text-only messages, let polling handle the rendering
- const data = {
- text: message,
+ const data = {
+ text: message,
context,
- message_id: messageId
+ message_id: messageId
};
response = await fetch('/msg', {
method: 'POST',
@@ -149,16 +150,16 @@ sendButton.addEventListener('click', sendMessage);
export function updateChatInput(text) {
console.log('updateChatInput called with:', text);
-
+
// Append text with proper spacing
const currentValue = chatInput.value;
const needsSpace = currentValue.length > 0 && !currentValue.endsWith(' ');
chatInput.value = currentValue + (needsSpace ? ' ' : '') + text + ' ';
-
+
// Adjust height and trigger input event
adjustTextareaHeight();
chatInput.dispatchEvent(new Event('input'));
-
+
console.log('Updated chat input value:', chatInput.value);
}
@@ -235,7 +236,7 @@ async function handleFileUpload(event) {
}
-window.loadKnowledge = async function() {
+window.loadKnowledge = async function () {
const input = document.createElement('input');
input.type = 'file';
input.accept = '.txt,.pdf,.csv,.html,.json,.md';
@@ -289,17 +290,17 @@ const workDirModalProxy = {
};
// Make the proxy available globally
-window.workDirModalProxy = workDirModalProxy;
+window.workDirModalProxy = workDirModalProxy;
// Ensure correct setup for Alpine.js x-data.
-window.workDirModal = function() {
+window.workDirModal = function () {
return workDirModalProxy; // Returns the proxy object for the Work Dir modal
}
document.addEventListener('alpine:init', () => {
// Make workDirModalProxy available as an Alpine component/store
- Alpine.data('workDirModal', workDirModal);
+ Alpine.data('workDirModal', workDirModal);
});
@@ -334,6 +335,7 @@ function generateGUID() {
let lastLogVersion = 0;
let lastLogGuid = ""
+let lastSpokenNo = 0
async function poll() {
let updated = false
@@ -356,6 +358,7 @@ async function poll() {
for (const log of response.logs) {
setMessage(log.no, log.type, log.heading, log.content, log.temp, log.kvps);
}
+ afterMessagesUpdate(response.logs)
}
updateProgress(response.log_progress)
@@ -383,6 +386,26 @@ async function poll() {
return updated
}
+function afterMessagesUpdate(logs) {
+ if(localStorage.getItem('speech') == 'true') {
+ speakMessages(logs)
+ }
+}
+
+function speakMessages(logs) {
+ // log.no, log.type, log.heading, log.content
+ for (let i = logs.length - 1; i >= 0; i--) {
+ const log = logs[i]
+ if (log.type == "response") {
+ if (log.no > lastSpokenNo) {
+ lastSpokenNo = log.no
+ speech.speak(log.content)
+ return
+ }
+ }
+ }
+}
+
function updateProgress(progress) {
if (!progress) progress = "Waiting for input"
@@ -453,6 +476,7 @@ const setContext = function (id) {
context = id
lastLogGuid = ""
lastLogVersion = 0
+ lastSpokenNo = 0
const chatsAD = Alpine.$data(chatsSection);
chatsAD.selected = id
}
@@ -488,6 +512,12 @@ window.toggleDarkMode = function (isDark) {
localStorage.setItem('darkMode', isDark);
};
+window.toggleSpeech = function (isOn) {
+ console.log("Speech:", isOn);
+ localStorage.setItem('speech', isOn);
+ if(!isOn) speech.stop()
+};
+
// Modify this part
document.addEventListener('DOMContentLoaded', () => {
const isDarkMode = localStorage.getItem('darkMode') !== 'false';
@@ -576,20 +606,20 @@ window.saveChat = async function () {
function downloadFile(filename, content) {
// Create a Blob with the content to save
const blob = new Blob([content], { type: 'application/json' });
-
+
// Create a link element
const link = document.createElement('a');
-
+
// Create a URL for the Blob
const url = URL.createObjectURL(blob);
link.href = url;
-
+
// Set the file name for download
link.download = filename;
-
+
// Programmatically click the link to trigger the download
link.click();
-
+
// Clean up by revoking the object URL
setTimeout(() => {
URL.revokeObjectURL(url);
diff --git a/webui/messages.js b/webui/messages.js
index 8e45a1137..a2639d8e5 100644
--- a/webui/messages.js
+++ b/webui/messages.js
@@ -62,10 +62,10 @@ export function _drawMessage(messageContainer, heading, content, temp, followUp,
if (window.renderMathInElement) {
renderMathInElement(spanElement, {
delimiters: [
- {left: "$", right: "$", display: true},
- {left: "\\$", right: "\\$", display: true},
- {left: "$", right: "$", display: false},
- {left: "\\$", right: "\\$", display: false}
+ { left: "$", right: "$", display: true },
+ { left: "\\$", right: "\\$", display: true },
+ { left: "$", right: "$", display: false },
+ { left: "\\$", right: "\\$", display: false }
],
throwOnError: false // Prevent KaTeX from throwing errors
});
@@ -115,7 +115,7 @@ export function drawMessageUser(messageContainer, id, type, heading, content, te
if (hasAttachments) {
const attachmentsContainer = document.createElement('div');
attachmentsContainer.classList.add('attachments-container');
-
+
// Add "Attachments" heading if there's text content
if (hasContent) {
const attachmentsHeading = document.createElement('h4');
@@ -204,10 +204,10 @@ function drawKvps(container, kvps) {
if (window.renderMathInElement) {
renderMathInElement(span, {
delimiters: [
- {left: "$$", right: "$$", display: true},
- {left: "\$$", right: "\$$", display: true},
- {left: "$", right: "$", display: false},
- {left: "\$$", right: "\$$", display: false}
+ { left: "$$", right: "$$", display: true },
+ { left: "\$$", right: "\$$", display: true },
+ { left: "$", right: "$", display: false },
+ { left: "\$$", right: "\$$", display: false }
],
throwOnError: false // Prevent KaTeX from throwing errors
});
diff --git a/webui/speech copy.js b/webui/speech copy.js
deleted file mode 100644
index eb2050075..000000000
--- a/webui/speech copy.js
+++ /dev/null
@@ -1,337 +0,0 @@
-import { pipeline, read_audio } from './[email protected]';
-import { updateChatInput, sendMessage } from './index.js';
-
-const microphoneButton = document.getElementById('microphone-button');
-let microphoneInput = null;
-let isProcessingClick = false;
-
-class MicrophoneInput {
- /**
- * Voice Input Handler with Whisper Transcription
- *
- * Whisper Model Size Configuration:
- * - 'tiny': Smallest model, fastest, lowest accuracy (~32MB)
- * - Best for: Quick prototyping, low-resource environments
- * - Pros: Very fast, low memory usage
- * - Cons: Lowest transcription accuracy
- *
- * - 'base': Small model, good balance of speed and accuracy (~74MB)
- * - Best for: General-purpose voice input
- * - Pros: Reasonable accuracy, moderate resource usage
- * - Cons: Less accurate than larger models
- *
- * - 'small': Medium-sized model, better accuracy (~244MB)
- * - Best for: More precise transcription needs
- * - Pros: Improved accuracy over base model
- * - Cons: Slower, more memory-intensive
- *
- * - 'medium': Large model with high accuracy (~769MB)
- * - Best for: Professional transcription, multi-language support
- * - Pros: Very high accuracy
- * - Cons: Significant computational resources required
- *
- * - 'large': Largest model, highest accuracy (~1.5GB)
- * - Best for: Professional, multi-language transcription
- * - Pros: Highest possible accuracy
- * - Cons: Slowest, most resource-intensive
- *
- * Recommended Default: 'base' for most web applications
- */
- constructor(updateCallback, options = {}) {
- this.mediaRecorder = null;
- this.audioChunks = [];
- this.isRecording = false;
- this.updateCallback = updateCallback;
- this.isFinalizing = false;
- this.messageSent = false; // move messageSent into class
-
- // New properties for silence detection
- this.audioContext = null;
- this.mediaStreamSource = null;
- this.analyserNode = null;
- this.silenceTimer = null;
- this.silenceThreshold = options.silenceThreshold || 0.01; // Adjust as needed
- this.silenceDuration = options.silenceDuration || 2000; // Duration in milliseconds
-
- this.options = {
- modelSize: 'tiny',
- language: 'en',
- chunkDuration: 3000,
- ...options
- };
- }
-
- async initialize() {
- try {
-
- this.transcriber = await pipeline(`automatic-speech-recognition`, `Xenova/whisper-${this.options.modelSize}.${this.options.language}`);
-
- const stream = await navigator.mediaDevices.getUserMedia({
- audio: {
- echoCancellation: true,
- noiseSuppression: true,
- channelCount: 1
- }
- });
-
- // Configure MediaRecorder
- this.mediaRecorder = new MediaRecorder(stream);
-
- // Handle audio data availability
- this.mediaRecorder.ondataavailable = async (event) => {
- if (event.data.size > 0) {
- this.audioChunks.push(event.data);
- // await this.processAudioChunk(event.data);
- }
- };
-
- // Handle recording stop
- this.mediaRecorder.onstop = async () => {
- await this.finalizeRecording();
- };
-
- // Set up AudioContext and AnalyserNode for silence detection
- this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
- this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
- this.analyserNode = this.audioContext.createAnalyser();
- this.analyserNode.minDecibels = -90;
- this.analyserNode.maxDecibels = -10;
- this.analyserNode.smoothingTimeConstant = 0.85;
-
- this.mediaStreamSource.connect(this.analyserNode);
- } catch (error) {
- console.error('Microphone initialization error:', error);
- toast('Failed to access microphone. Please check permissions.', 'error');
- }
- }
-
- startSilenceDetection() {
- const dataArray = new Uint8Array(this.analyserNode.fftSize);
- const checkSilence = () => {
- this.analyserNode.getByteTimeDomainData(dataArray);
-
- let sum = 0;
- for (let i = 0; i < dataArray.length; i++) {
- const amplitude = (dataArray[i] - 128) / 128;
- sum += amplitude * amplitude;
- }
- const rms = Math.sqrt(sum / dataArray.length);
-
- if (rms < this.silenceThreshold) {
- if (!this.silenceTimer) {
- this.silenceTimer = setTimeout(() => {
- if (this.isRecording) {
- console.log('Silence detected. Stopping recording.');
- this.stopRecording();
- microphoneButton.classList.remove('recording');
- microphoneButton.classList.remove('mic-pulse');
- }
- }, this.silenceDuration);
- }
- } else {
- if (this.silenceTimer) {
- clearTimeout(this.silenceTimer);
- this.silenceTimer = null;
- }
- }
-
- if (this.isRecording) {
- requestAnimationFrame(checkSilence);
- }
- };
-
- if (this.isRecording) {
- requestAnimationFrame(checkSilence);
- }
- }
-
- startRecording() {
- if (this.mediaRecorder && this.audioContext) {
- this.isRecording = true;
- this.audioChunks = [];
- this.messageSent = false;
- this.mediaRecorder.start(this.options.chunkDuration);
- this.audioContext.resume();
- this.startSilenceDetection();
- }
- }
-
- stopRecording() {
- if (this.mediaRecorder && this.isRecording) {
- this.isRecording = false;
- if (!this.isFinalizing) {
- this.isFinalizing = true;
- this.mediaRecorder.stop();
- this.audioContext.suspend();
- if (this.silenceTimer) {
- clearTimeout(this.silenceTimer);
- this.silenceTimer = null;
- }
- }
- }
- }
-
-
- async finalizeRecording() {
- if (this.isFinalizing) {
- this.isFinalizing = false;
-
- if (this.audioChunks.length > 0) {
-
- const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });
- const audioUrl = URL.createObjectURL(audioBlob);
- const samplingRate = 16000; // Adjust as needed for the model
- const audioData = await read_audio(audioUrl, samplingRate);
- URL.revokeObjectURL(audioUrl);
-
- // Transcribe the audio
- const result = await this.transcriber(audioData);
-
- if (result.text) {
- console.log('Final transcription received:', result.text);
- await this.updateCallback(result.text, true);
- } else {
- console.warn('Final transcription returned empty text.');
- }
-
-
- // Release the object URL after use
-
- // const audioBlob = new Blob(this.audioChunks, { type: 'audio/webm' });
- // this.audioChunks = []; // Clear for next recording
-
- // const reader = new FileReader();
- // reader.onloadend = async () => {
- // const base64Data = reader.result.split(',')[1];
-
- // try {
- // const response = await fetch('/transcribe', {
- // method: 'POST',
- // headers: {
- // 'Content-Type': 'application/json'
- // },
- // body: JSON.stringify({
- // audio_data: base64Data,
- // model_size: this.options.modelSize,
- // language: this.options.language,
- // is_final: true
- // })
- // });
-
- // const result = await response.json();
-
- // if (result.text) {
- // console.log('Final transcription received:', result.text);
- // await this.updateCallback(result.text, true);
- // } else {
- // console.warn('Final transcription returned empty text.');
- // }
- // } catch (transcribeError) {
- // console.error('Final transcription error:', transcribeError);
- // toast('Final transcription failed.', 'error');
- // } finally {
- // // Reset the microphone button state
- // microphoneButton.classList.remove('recording');
- // microphoneButton.classList.remove('mic-pulse');
- // microphoneButton.style.backgroundColor = '';
- // }
- // };
- // reader.readAsDataURL(audioBlob);
- }
- }
- }
-}
-
-export default MicrophoneInput;
-
-async function initializeMicrophoneInput() {
- console.log('Initializing microphone input');
-
- microphoneInput = new MicrophoneInput(
- async (text, isFinal) => {
- if (isFinal) {
- console.log('Final transcription callback received:', text);
- updateChatInput(text)
- // chatInput.value = text;
- // adjustTextareaHeight();
-
- if (!microphoneInput.messageSent) {
- microphoneInput.messageSent = true;
- console.log('Sending message');
- await sendMessage();
- }
- }
- },
- {
- modelSize: 'tiny',
- language: 'en',
- silenceThreshold: 0.07, // Adjust as needed
- silenceDuration: 2000, // Adjust as needed
- onError: (error) => {
- console.error('Microphone input error:', error);
- toast('Microphone error: ' + error.message, 'error');
- // Reset recording state
- if (microphoneButton.classList.contains('recording')) {
- microphoneButton.classList.remove('recording');
- }
- }
- }
- );
-
- await microphoneInput.initialize();
-}
-
-
-function toggleRecording() {
- console.log('toggleRecording called, isRecording:', microphoneInput.isRecording);
-
- if (microphoneInput.isRecording) {
- microphoneInput.stopRecording();
- microphoneButton.classList.remove('recording');
- // Add pulsing animation class
- microphoneButton.classList.remove('mic-pulse');
- } else {
- microphoneInput.startRecording();
- microphoneButton.classList.add('recording');
- // Add pulsing animation class
- microphoneButton.classList.add('mic-pulse');
- }
-
- // Add visual feedback
- microphoneButton.style.backgroundColor = microphoneInput.isRecording ? '#ff4444' : '';
- console.log('New recording state:', microphoneInput.isRecording);
-}
-
-// Some error handling for microphone input
-async function requestMicrophonePermission() {
- try {
- await navigator.mediaDevices.getUserMedia({ audio: true });
- return true;
- } catch (err) {
- console.error('Error accessing microphone:', err);
- toast('Microphone access denied. Please enable microphone access in your browser settings.', 'error');
- return false;
- }
-}
-// microphoneButton click event listener modifier
-microphoneButton.addEventListener('click', async () => {
- console.log('Microphone button clicked');
- if (isProcessingClick) {
- console.log('Click already being processed, ignoring');
- return;
- }
- isProcessingClick = true;
-
- const hasPermission = await requestMicrophonePermission();
- if (!hasPermission) return;
-
- if (!microphoneInput) {
- await initializeMicrophoneInput();
- }
-
- await toggleRecording();
-
- setTimeout(() => {
- isProcessingClick = false;
- }, 300); // Add a 300ms delay before allowing another click
-});
diff --git a/webui/speech.js b/webui/speech.js
index f7dd0055d..69c545fb7 100644
--- a/webui/speech.js
+++ b/webui/speech.js
@@ -217,7 +217,8 @@ class MicrophoneInput {
this.silenceStartTime = null;
if (this.status === Status.LISTENING || this.status === Status.WAITING) {
- this.status = Status.RECORDING;
+ if (!speech.isSpeaking()) // TODO? a better way to ignore agent's voice?
+ this.status = Status.RECORDING;
}
} else if (this.status === Status.RECORDING) {
if (!this.silenceStartTime) {
@@ -347,3 +348,44 @@ async function requestMicrophonePermission() {
return false;
}
}
+
+
+class Speech {
+ constructor() {
+ this.synth = window.speechSynthesis;
+ this.utterance = null;
+ }
+
+ stripEmojis(str) {
+ return str
+ .replace(/([\u2700-\u27BF]|[\uE000-\uF8FF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDFFF]|[\u2011-\u26FF]|\uD83E[\uDD10-\uDDFF])/g, '')
+ .replace(/\s+/g, ' ')
+ .trim();
+ }
+
+ speak(text) {
+ console.log('Speaking:', text);
+ // Stop any current utterance
+ this.stop();
+
+ // Remove emojis and create a new utterance
+ text = this.stripEmojis(text);
+ this.utterance = new SpeechSynthesisUtterance(text);
+
+ // Speak the new utterance
+ this.synth.speak(this.utterance);
+ }
+
+ stop() {
+ if (this.isSpeaking()) {
+ this.synth.cancel();
+ }
+ }
+
+ isSpeaking() {
+ return this.synth?.speaking || false;
+ }
+}
+
+export const speech = new Speech();
+window.speech = speech
\ No newline at end of file
diff --git a/webui/test.html b/webui/test.html
deleted file mode 100644
index 7856867fb..000000000
--- a/webui/test.html
+++ /dev/null
@@ -1,82 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
- <meta charset="UTF-8">
- <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
- <title>Agent Zero</title>
-
- <script type="module">
- import { pipeline, read_audio } from './[email protected]';
-
- let transcriber;
- let mediaRecorder;
- let audioChunks = [];
- let isRecording = false;
-
- // Initialize the transcriber
- async function initTranscriber() {
- transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');
- }
-
- // Toggle recording state
- async function toggleRecording() {
- if (isRecording) {
- stopRecording();
- } else {
- startRecording();
- }
- }
-
- // Start recording from the microphone
- async function startRecording() {
- isRecording = true;
- audioChunks = [];
- document.getElementById("micButton").innerText = "Stop Recording";
-
- const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
- mediaRecorder = new MediaRecorder(stream);
-
- mediaRecorder.ondataavailable = event => {
- audioChunks.push(event.data);
- };
-
- mediaRecorder.onstop = async () => {
- const audioBlob = new Blob(audioChunks, { type: 'audio/wav' });
- const audioUrl = URL.createObjectURL(audioBlob);
-
- // Use read_audio to fetch and process the audio from the URL
- const samplingRate = 16000; // Adjust as needed for the model
- const audioData = await read_audio(audioUrl, samplingRate);
-
- // Transcribe the audio
- const output = await transcriber(audioData);
- document.getElementById("transcript").innerText = output.text;
-
- // Release the object URL after use
- URL.revokeObjectURL(audioUrl);
- };
-
- mediaRecorder.start();
- }
-
- // Stop recording
- function stopRecording() {
- isRecording = false;
- document.getElementById("micButton").innerText = "Start Recording";
- mediaRecorder.stop();
- }
-
- // Expose functions to the global scope
- window.toggleRecording = toggleRecording;
-
- // Initialize the transcriber on page load
- window.onload = initTranscriber;
- </script>
-</head>
-
-<body>
- <h1>Agent Zero Speech Transcription</h1>
- <button id="micButton" onclick="toggleRecording()">Start Recording</button>
- <p id="transcript">Transcript will appear here...</p>
-</body>
-</html>
diff --git a/webui/test2.html b/webui/test2.html
deleted file mode 100644
index 02e431458..000000000
--- a/webui/test2.html
+++ /dev/null
@@ -1,122 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
- <meta charset="UTF-8">
- <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
- <title>Agent Zero</title>
-
- <script type="module">
- import { pipeline, read_audio } from './[email protected]';
-
- let transcriber;
- let mediaRecorder;
- let isRecording = false;
- let audioChunks = [];
- let chunks_to_process = [{ tokens: [], finalized: false }];
-
- async function initTranscriber() {
- try {
- transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');
- } catch (error) {
- console.error("Failed to initialize transcriber:", error);
- }
- }
-
- async function toggleRecording() {
- if (isRecording) {
- stopRecording();
- } else {
- startRecording();
- }
- }
-
- async function startRecording() {
- isRecording = true;
- document.getElementById("micButton").innerText = "Stop Recording";
-
- const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
- mediaRecorder = new MediaRecorder(stream, { mimeType: 'audio/webm;codecs=opus' });
-
- // Handle data as it’s available
- mediaRecorder.ondataavailable = (event) => {
- audioChunks.push(event.data);
-
- // Process chunks every 3 seconds
- if (audioChunks.length >= 3) {
- processAudioChunks();
- }
- };
-
- // Capture small chunks every second
- mediaRecorder.start(1000);
- }
-
- async function processAudioChunks() {
- const audioBlob = new Blob(audioChunks, { type: 'audio/webm' });
- const audioUrl = URL.createObjectURL(audioBlob);
-
- try {
- const audioData = await read_audio(audioUrl, 16000);
-
- if (transcriber) {
- await transcriber(audioData, {
- callback_function: mergeChunksCallback,
- chunk_callback: processChunkCallback,
- return_timestamps: true,
- force_full_sequences: false,
- });
- } else {
- console.warn("Transcriber is not ready yet.");
- }
- } catch (error) {
- console.error("Error during transcription:", error);
- }
-
- // Cleanup and reset buffer
- URL.revokeObjectURL(audioUrl);
- audioChunks = [];
- }
-
- function processChunkCallback(chunk) {
- let lastChunk = chunks_to_process[chunks_to_process.length - 1];
- Object.assign(lastChunk, chunk);
- lastChunk.finalized = true;
-
- if (!chunk.is_last) {
- chunks_to_process.push({ tokens: [], finalized: false });
- }
- }
-
- function mergeChunksCallback(item) {
- let lastChunk = chunks_to_process[chunks_to_process.length - 1];
- lastChunk.tokens = [...item[0].output_token_ids];
-
- // Merge text chunks and update transcript
- const mergedText = transcriber.tokenizer._decode_asr(chunks_to_process, {
- time_precision: 1.0 / transcriber.processor.feature_extractor.config.chunk_length,
- return_timestamps: true,
- force_full_sequences: false,
- });
-
- document.getElementById("transcript").innerText += mergedText + ' ';
- }
-
- function stopRecording() {
- isRecording = false;
- document.getElementById("micButton").innerText = "Start Recording";
- if (mediaRecorder && mediaRecorder.state !== 'inactive') {
- mediaRecorder.stop();
- }
- }
-
- window.toggleRecording = toggleRecording;
- window.onload = initTranscriber;
- </script>
-</head>
-
-<body>
- <h1>Agent Zero Speech Transcription</h1>
- <button id="micButton" onclick="toggleRecording()">Start Recording</button>
- <p id="transcript">Transcript will appear here...</p>
-</body>
-</html>