Agent Zero AI framework
speech chunking improvements
frdele33f6e455be2
diff --git a/webui/components/chat/speech/speech-store.js b/webui/components/chat/speech/speech-store.jsindex 8e39fb423..18b897b84 100644--- a/webui/components/chat/speech/speech-store.js+++ b/webui/components/chat/speech/speech-store.js@@ -193,6 +193,8 @@ const model = {this.ttsStream.chunks = this.chunkText(cleanText);if (this.ttsStream.chunks.length == 0) return;+ console.log("chunks updated", JSON.stringify(cleanText), this.ttsStream.chunks)+// if stream was already running, just updating chunks is enoughif (this.ttsStream.running) return;else this.ttsStream.running = true; // proceed to running phase@@ -201,6 +203,8 @@ const model = {const terminator = () =>this.ttsStream?.id !== id || this.ttsStream?.stopped;+ const spoken = []+// loop chunks from last spoken chunk indexfor (let i = this.ttsStream.lastChunkIndex + 1;@@ -215,8 +219,10 @@ const model = {this.ttsStream.lastChunkIndex = i;// speak the chunk+ spoken.push(this.ttsStream.chunks[i]);await this._speak(this.ttsStream.chunks[i], i > 0, () => terminator());}+ console.log("finished speaking", spoken)// at the end, finish stream datathis.ttsStream.running = false;@@ -245,18 +251,8 @@ const model = {chunkText(text, { maxChunkLength = 135, lineSeparator = "..." } = {}) {const INC_LIMIT = maxChunkLength * 2;- const chunks = [];- let buffer = "";-- // Helper to push chunk if not empty- const push = (s) => {- if (s) chunks.push(s.trimEnd());- };- const flush = () => {- push(buffer);- buffer = "";- };-+ const MIN_CHUNK_LENGTH = 20; // minimum length for a chunk before merging+// Only split by ,/word if needed (unchanged)const splitDeep = (seg) => {if (seg.length <= INC_LIMIT) return [seg];@@ -272,7 +268,7 @@ const model = {if (need <= maxChunkLength) {part += (part ? " " : "") + word;} else {- push(part);+ if (part) out.push(part);if (word.length > maxChunkLength) {for (let i = 0; i < word.length; i += maxChunkLength)out.push(word.slice(i, i + maxChunkLength));@@ -282,7 +278,7 @@ const model = {}}}- push(part);+ if (part) out.push(part);return out;};@@ -304,33 +300,62 @@ const model = {if (start < line.length) toks.push(line.slice(start));return toks;};-- // --- main loop: JOIN lines with separator *only if they fit in buffer* ---- const lines = text.split(/\n+/).filter((l) => l.trim());- for (let i = 0; i < lines.length; ++i) {- const line = lines[i].trim();- if (!line) continue;- // Expand line into sentence tokens and join them back, so only lines are joined with separator- const sentenceStr = sentenceTokens(line).join(" ");-- // If buffer is empty, just start with the line- if (!buffer) {- buffer = sentenceStr;- } else {- // Try joining the line with separator- const join = buffer + " " + lineSeparator + " " + sentenceStr;- if (join.length <= maxChunkLength) {- buffer = join;++ // Step 1: Split all newlines into individual chunks first+ let initialChunks = [];+ const lines = text.split(/\n+/).filter(l => l.trim());++ for (const line of lines) {+ if (!line.trim()) continue;+ // Process each line into sentence tokens and add to chunks+ const sentenceStr = sentenceTokens(line.trim()).join(" ");+ initialChunks.push(sentenceStr);+ }++ // Step 2: Merge short chunks until they meet minimum length criteria+ const finalChunks = [];+ let currentChunk = "";++ for (let i = 0; i < initialChunks.length; i++) {+ const chunk = initialChunks[i];++ // If current chunk is empty, start with this chunk+ if (!currentChunk) {+ currentChunk = chunk;+ // If this is the last chunk or it's already long enough, add it+ if (i === initialChunks.length - 1 || currentChunk.length >= MIN_CHUNK_LENGTH) {+ finalChunks.push(currentChunk);+ currentChunk = "";+ }+ continue;+ }++ // Current chunk exists, check if we should merge+ if (currentChunk.length < MIN_CHUNK_LENGTH) {+ // Try to merge with separator+ const merged = currentChunk + " " + lineSeparator + " " + chunk;++ // Check if merged chunk fits within max length+ if (merged.length <= maxChunkLength) {+ currentChunk = merged;} else {- // Flush buffer, start new chunk with this line- flush();- buffer = sentenceStr;+ // Doesn't fit, add current chunk and start new one+ finalChunks.push(currentChunk);+ currentChunk = chunk;}+ } else {+ // Current chunk is already long enough, add it and start new one+ finalChunks.push(currentChunk);+ currentChunk = chunk;+ }++ // If this is the last chunk, add whatever is in the buffer+ if (i === initialChunks.length - 1 && currentChunk) {+ finalChunks.push(currentChunk);}}- flush();-- return chunks;++ return finalChunks.map(chunk => chunk.trimEnd());},// Show a prompt to user to enable audio@@ -453,18 +478,59 @@ const model = {// Clean text for TTScleanText(text) {- // kokoro can have trouble speaking short list items, so we group them them- text = joinShortMarkdownLists(text);- // Remove code blocks: ```...```- text = text.replace(/```[\s\S]*?```/g, "");- // Remove inline code ticks: `...`+ // Use SUB character (ASCII 26, 0x1A) for placeholders to avoid conflicts with actual text+ const SUB = "\x1A"; // non-printable substitute character+ const codePlaceholder = SUB + "code" + SUB;+ const tablePlaceholder = SUB + "table" + SUB;++ // Helper function to handle both closed and unclosed patterns+ // replacement can be a string or null (to remove)+ function handlePatterns(inputText, closedPattern, unclosedPattern, replacement) {+ // Process closed patterns first+ let processed = inputText.replace(closedPattern, replacement || "");++ // If the text changed, it means we found and replaced closed patterns+ if (processed !== inputText) {+ return processed;+ } else {+ // No closed patterns found, check for unclosed ones+ const unclosedMatch = inputText.match(unclosedPattern);+ if (unclosedMatch) {+ // Replace the unclosed pattern+ return inputText.replace(unclosedPattern, replacement || "");+ }+ }++ // No patterns found, return original+ return inputText;+ }++ // Handle code blocks+ text = handlePatterns(+ text,+ /```(?:[a-zA-Z0-9]*\n)?[\s\S]*?```/g, // closed code blocks+ /```(?:[a-zA-Z0-9]*\n)?[\s\S]*$/g, // unclosed code blocks+ codePlaceholder+ );++ // Replace inline code ticks with content preservedtext = text.replace(/`([^`]*)`/g, "$1"); // remove backticks but keep content- // Remove HTML tags and their content: <tag>content</tag>- text = text.replace(/<[a-zA-Z][a-zA-Z0-9]*>.*?<\/[a-zA-Z][a-zA-Z0-9]*>/gs, "");+ // Handle HTML tags+ text = handlePatterns(+ text,+ /<[a-zA-Z][a-zA-Z0-9]*>.*?<\/[a-zA-Z][a-zA-Z0-9]*>/gs, // closed HTML tags+ /<[a-zA-Z][a-zA-Z0-9]*>[\s\S]*$/g, // unclosed HTML tags+ "" // remove HTML tags completely+ );- // Remove self-closing HTML tags: <tag/>- text = text.replace(/<[a-zA-Z][a-zA-Z0-9]*(\/| [^>]*\/>)/g, "");+ // Handle self-closing HTML tags+ text = handlePatterns(+ text,+ /<[a-zA-Z][a-zA-Z0-9]*(\/| [^>]*\/>)/g, // complete self-closing tags+ /<[a-zA-Z][a-zA-Z0-9]* [^>]*$/g, // incomplete self-closing tags+ ""+ );// Remove markdown links: [label](url) → labeltext = text.replace(/\[([^\]]+)\]\([^\)]+\)/g, "$1");@@ -472,8 +538,21 @@ const model = {// Remove markdown formatting: *, _, #text = text.replace(/[*_#]+/g, "");- // Remove tables (basic): lines with |...|- text = text.replace(/\|[^\n]*\|/g, "");+ // Handle tables - both complete and partial+ // Check if text contains a table-like pattern+ if (text.includes("|")) {+ // Find consecutive lines with | characters (table rows)+ const tableLines = text.split("\n").filter(line => line.includes("|") && line.trim().startsWith("|"));+ if (tableLines.length > 0) {+ // Replace each table line with a placeholder+ for (const line of tableLines) {+ text = text.replace(line, tablePlaceholder);+ }+ } else {+ // Just handle individual table rows+ text = text.replace(/\|[^\n]*\|/g, tablePlaceholder);+ }+ }// Remove emojis and private unicode blockstext = text.replace(@@ -490,35 +569,6 @@ const model = {}});- // kokoro can have trouble speaking short list items, so we group them them- function joinShortMarkdownLists(txt, minItemLength = 40) {- const lines = txt.split(/\r?\n/);- const newLines = [];- let buffer = [];- const isShortList = (line) =>- /^\s*-\s+/.test(line) && line.trim().length < minItemLength;- for (let i = 0; i < lines.length; i++) {- if (isShortList(lines[i])) {- buffer.push(lines[i].replace(/^\s*-\s+/, "").trim());- } else {- if (buffer.length > 1) {- newLines.push(buffer.join(", "));- buffer = [];- } else if (buffer.length === 1) {- newLines.push(buffer[0]);- buffer = [];- }- newLines.push(lines[i]);- }- }- if (buffer.length > 1) {- newLines.push(buffer.join(", "));- } else if (buffer.length === 1) {- newLines.push(buffer[0]);- }- return newLines.join("\n");- }-// Remove email addresses// text = text.replace(/\S+@\S+/g, "");@@ -530,6 +580,22 @@ const model = {// Collapse multiple spaces/tabs to a single space, but preserve newlinestext = text.replace(/[ \t]+/g, " ");++ // Function to merge consecutive placeholders of any type+ function mergePlaceholders(txt, placeholder, replacement) {+ // Create regex for consecutive placeholders (with possible whitespace between)+ const regex = new RegExp(placeholder + "\\s*" + placeholder, "g");+ // Merge consecutive placeholders until no more found+ while (regex.test(txt)) {+ txt = txt.replace(regex, placeholder);+ }+ // Replace all remaining placeholders with human-readable text+ return txt.replace(new RegExp(placeholder, "g"), replacement);+ }++ // Apply placeholder merging for both types+ text = mergePlaceholders(text, codePlaceholder, "See code attached ...");+ text = mergePlaceholders(text, tablePlaceholder, "See table attached ...");// Trim leading/trailing whitespacetext = text.trim();