Agent Zero AI framework
Add Telegram speech mode
Anmol Malik709b86539965
diff --git a/helpers/integration_commands.py b/helpers/integration_commands.pyindex e4a09b32b..b555d2af0 100644--- a/helpers/integration_commands.py+++ b/helpers/integration_commands.py@@ -70,6 +70,12 @@ COMMAND_REGISTRY: tuple[IntegrationCommandDef, ...] = ("Configuration",args_hint="[on|off]",),+ IntegrationCommandDef(+ "speech",+ "Enable or disable Telegram spoken replies.",+ "Configuration",+ args_hint="[on|off]",+ ),IntegrationCommandDef("project","Show or switch the active project.",@@ -194,6 +200,8 @@ def try_handle_command(context: "AgentContext", text: str) -> str | None:return _handle_toggle(context, args, "telegram_stream_enabled", "Response streaming")if command == "/tools":return _handle_toggle(context, args, "telegram_tools_enabled", "Tool progress")+ if command == "/speech":+ return _handle_toggle(context, args, "telegram_speech_enabled", "Speech replies")if command == "/project":return _handle_project(context, args)if command == "/model":diff --git a/plugins/_telegram_integration/README.md b/plugins/_telegram_integration/README.mdindex a9793fa35..c3c13aae2 100644--- a/plugins/_telegram_integration/README.md+++ b/plugins/_telegram_integration/README.md@@ -24,6 +24,7 @@ This plugin connects one or more Telegram bots to Agent Zero. Each bot runs inde- `/agent <profile>` switches the active agent profile when the current run is idle.- `/model`, `/project`, and `/agent` show Telegram inline keyboard pickers when used without arguments.- `/stream` toggles live response streaming; `/tools` toggles tool progress messages.+ - `/speech` toggles spoken replies. When enabled, final text replies also include Kokoro-generated Telegram audio when Kokoro TTS is available.- `/send` or `/queue send` flushes queued messages for the current chat.- `/steer <message>` sends an intervention to the active run.- **Group support**@@ -35,6 +36,7 @@ This plugin connects one or more Telegram bots to Agent Zero. Each bot runs inde- **Reply delivery**- Streams tool progress and response text through real Telegram messages updated with `editMessageText`.- Tool progress and the AI response are separate messages; only the AI response replies to the user message.+ - Sends native Telegram media: photos, voice/audio files, videos, and document fallback.- `tool_execute_after` intercepts the `response` tool — sends `break_loop=false` updates as separate intermediate Telegram messages.- `process_chain_end` auto-sends the final response, with retry logic on failure.- **Formatting**@@ -63,6 +65,7 @@ This plugin connects one or more Telegram bots to Agent Zero. Each bot runs inde- `helpers/telegram_client.py` — Low-level Telegram API wrapper: send text/file/photo, Markdown→HTML converter, keyboard builder, message splitting.- `helpers/command_ui.py` — Telegram inline keyboard command pickers and callback handling.- `helpers/draft_stream.py` — Editable Telegram message streaming for tool progress and response previews.+ - `helpers/speech.py` — Telegram speech-mode TTS synthesis and audio attachment generation.- **Extensions**- `extensions/python/job_loop/_10_telegram_bot.py` — Bot lifecycle manager, starts/stops bots on each tick.- `extensions/python/message_loop_start/_45_telegram_draft_start.py` — Initializes Telegram response streaming.diff --git a/plugins/_telegram_integration/extensions/python/process_chain_end/_55_telegram_reply.py b/plugins/_telegram_integration/extensions/python/process_chain_end/_55_telegram_reply.pyindex 508c3acf4..582d444b2 100644--- a/plugins/_telegram_integration/extensions/python/process_chain_end/_55_telegram_reply.py+++ b/plugins/_telegram_integration/extensions/python/process_chain_end/_55_telegram_reply.py@@ -32,7 +32,17 @@ class TelegramAutoReply(Extension):try:if response_text:+ from plugins._telegram_integration.helpers import speech++ speech_attachment, speech_warning = await speech.synthesize_attachment(+ context, response_text+ )+ if speech_attachment:+ attachments = [*attachments, speech_attachment]await self._send_reply(context, response_text, attachments, keyboard)+ warning = speech.consume_warning(context, speech_warning)+ if warning:+ await self._send_reply(context, warning, [], None)except Exception as e:PrintStyle.error(f"Telegram auto-reply error: {format_error(e)}")finally:diff --git a/plugins/_telegram_integration/helpers/command_ui.py b/plugins/_telegram_integration/helpers/command_ui.pyindex a0f108d0f..6ceb3fe6e 100644--- a/plugins/_telegram_integration/helpers/command_ui.py+++ b/plugins/_telegram_integration/helpers/command_ui.py@@ -13,6 +13,7 @@ from plugins._telegram_integration.helpers import telegram_client as tcfrom plugins._telegram_integration.helpers.constants import (CTX_TG_STREAM_ENABLED,CTX_TG_TOOLS_ENABLED,+ CTX_TG_SPEECH_ENABLED,CTX_TG_BOT,CTX_TG_CHAT_ID,CTX_TG_CHAT_TYPE,@@ -85,6 +86,17 @@ async def handle_command(args,)return True+ if command == "/speech":+ await send_toggle_picker(+ context,+ token,+ chat_id,+ reply_to_message_id,+ CTX_TG_SPEECH_ENABLED,+ "Speech replies",+ args,+ )+ return Truereturn False@@ -129,9 +141,8 @@ async def handle_callback(selected_context = await _select_session(context, _safe_int(value))await edit_session_picker(selected_context or context, token, chat_id, message_id, 0, selected=bool(selected_context))return True- if kind in {"stream", "tools"} and action in {"on", "off"}:- key = CTX_TG_STREAM_ENABLED if kind == "stream" else CTX_TG_TOOLS_ENABLED- label = "Response streaming" if kind == "stream" else "Tool progress"+ if kind in {"stream", "tools", "speech"} and action in {"on", "off"}:+ key, label = _toggle_key_label(kind)context.set_data(key, action == "on")save_tmp_chat(context)mark_dirty_for_context(context.id, reason=f"telegram.{kind}_toggle")@@ -334,7 +345,7 @@ def _agent_view(def _toggle_view(context: AgentContext, key: str, label: str) -> tuple[str, dict]:enabled = _toggle_enabled(context, key)- kind = "stream" if key == CTX_TG_STREAM_ENABLED else "tools"+ kind = _toggle_kind(key)state = "enabled" if enabled else "disabled"text = f"{_html(label)}: <b>{state}</b>"rows = [[@@ -542,9 +553,27 @@ def _paged_buttons(def _toggle_enabled(context: AgentContext, key: str) -> bool:value = context.get_data(key)+ if key == CTX_TG_SPEECH_ENABLED:+ return bool(value)return True if value is None else bool(value)+def _toggle_kind(key: str) -> str:+ if key == CTX_TG_STREAM_ENABLED:+ return "stream"+ if key == CTX_TG_TOOLS_ENABLED:+ return "tools"+ return "speech"+++def _toggle_key_label(kind: str) -> tuple[str, str]:+ if kind == "stream":+ return CTX_TG_STREAM_ENABLED, "Response streaming"+ if kind == "tools":+ return CTX_TG_TOOLS_ENABLED, "Tool progress"+ return CTX_TG_SPEECH_ENABLED, "Speech replies"++def _parse_toggle(args: str) -> bool | None:value = (args or "").strip().lower()if value in {"on", "enable", "enabled", "yes", "true", "1"}:diff --git a/plugins/_telegram_integration/helpers/constants.py b/plugins/_telegram_integration/helpers/constants.pyindex 190ee7fa4..9696f2661 100644--- a/plugins/_telegram_integration/helpers/constants.py+++ b/plugins/_telegram_integration/helpers/constants.py@@ -13,6 +13,7 @@ CTX_TG_TYPING_STOP = "_telegram_typing_stop"CTX_TG_REPLY_TO = "_telegram_reply_to_message_id"CTX_TG_STREAM_ENABLED = "telegram_stream_enabled"CTX_TG_TOOLS_ENABLED = "telegram_tools_enabled"+CTX_TG_SPEECH_ENABLED = "telegram_speech_enabled"# TransientCTX_TG_ATTACHMENTS = "_telegram_response_attachments"@@ -25,3 +26,4 @@ CTX_TG_RESPONSE_LAST_UPDATE = "_telegram_response_last_update"CTX_TG_ERROR_SENT = "_telegram_error_sent"CTX_TG_HEARTBEAT_TASK = "_telegram_heartbeat_task"CTX_TG_HEARTBEAT_STOP = "_telegram_heartbeat_stop"+CTX_TG_SPEECH_WARNING_SENT = "_telegram_speech_warning_sent"diff --git a/plugins/_telegram_integration/helpers/speech.py b/plugins/_telegram_integration/helpers/speech.pynew file mode 100644index 000000000..eb74a5e30--- /dev/null+++ b/plugins/_telegram_integration/helpers/speech.py@@ -0,0 +1,80 @@+from __future__ import annotations++import re+import uuid++from agent import AgentContext+from helpers import files+from plugins._telegram_integration.helpers.constants import (+ CTX_TG_SPEECH_ENABLED,+ CTX_TG_SPEECH_WARNING_SENT,+ DOWNLOAD_FOLDER,+)+++MAX_SPEECH_CHARS = 1800+++def is_enabled(context: AgentContext) -> bool:+ return bool(context.get_data(CTX_TG_SPEECH_ENABLED))+++async def synthesize_attachment(context: AgentContext, text: str) -> tuple[str | None, str | None]:+ if not is_enabled(context):+ return None, None++ speech_text = _speech_text(text)+ if not speech_text:+ return None, "Speech mode is enabled, but there was no readable text to speak."++ try:+ from plugins._kokoro_tts.helpers import runtime+ except Exception:+ return None, _setup_message()++ if not runtime.is_globally_enabled():+ return None, _setup_message()++ try:+ audio = await runtime.synthesize_sentences([speech_text])+ except Exception as exc:+ return None, f"Speech mode is enabled, but text-to-speech failed: {exc}"++ if not audio:+ return None, "Speech mode is enabled, but text-to-speech returned no audio."++ name = f"telegram_speech_{context.id}_{uuid.uuid4().hex[:10]}.wav"+ relative_path = f"{DOWNLOAD_FOLDER}/{name}"+ files.write_file_base64(relative_path, audio)+ return files.get_abs_path_dockerized(relative_path), None+++def consume_warning(context: AgentContext, warning: str | None) -> str | None:+ if not warning:+ context.data.pop(CTX_TG_SPEECH_WARNING_SENT, None)+ return None+ if context.data.get(CTX_TG_SPEECH_WARNING_SENT):+ return None+ context.data[CTX_TG_SPEECH_WARNING_SENT] = True+ return warning+++def _setup_message() -> str:+ return (+ "Speech mode is enabled, but Kokoro TTS is not available. "+ "Enable the Kokoro TTS plugin from Agent Zero settings, then try again."+ )+++def _speech_text(text: str) -> str:+ value = str(text or "")+ value = re.sub(r"```.*?```", " ", value, flags=re.DOTALL)+ value = re.sub(r"`([^`]+)`", r"\1", value)+ value = re.sub(r"!\[([^\]]*)\]\([^)]+\)", r"\1", value)+ value = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", value)+ value = re.sub(r"<[^>]+>", " ", value)+ value = re.sub(r"[*_#>\-]+", " ", value)+ value = re.sub(r"\s+", " ", value).strip()+ if len(value) > MAX_SPEECH_CHARS:+ value = value[:MAX_SPEECH_CHARS].rstrip()+ return valuediff --git a/tests/test_telegram_speech_mode.py b/tests/test_telegram_speech_mode.pynew file mode 100644index 000000000..575cc2777--- /dev/null+++ b/tests/test_telegram_speech_mode.py@@ -0,0 +1,112 @@+import asyncio++from plugins._telegram_integration.extensions.python.process_chain_end import (+ _55_telegram_reply,+)+from plugins._telegram_integration.helpers import command_ui, speech+from plugins._telegram_integration.helpers.constants import (+ CTX_TG_BOT,+ CTX_TG_REPLY_TO,+ CTX_TG_SPEECH_ENABLED,+)+++class FakeContext:+ id = "ctx-speech"++ def __init__(self):+ self.data = {}++ def get_data(self, key):+ return self.data.get(key)++ def set_data(self, key, value):+ self.data[key] = value+++class FakeAgent:+ number = 0++ def __init__(self):+ self.context = FakeContext()+ self.context.data[CTX_TG_BOT] = "main"+ self.context.data[CTX_TG_REPLY_TO] = 456+++def test_speech_toggle_defaults_off():+ context = FakeContext()++ text, markup = command_ui._toggle_view(context, CTX_TG_SPEECH_ENABLED, "Speech replies")++ assert "Speech replies: <b>disabled</b>" == text+ assert markup["inline_keyboard"][0][0]["callback_data"] == "tg:speech:on"+ assert markup["inline_keyboard"][0][1]["callback_data"] == "tg:speech:off"+++def test_speech_text_is_cleaned_for_tts():+ cleaned = speech._speech_text(+ "**Hello** [`Agent Zero`](https://example.com)\n\n```python\nprint('skip')\n```"+ )++ assert cleaned == "Hello Agent Zero"+++def test_speech_synthesis_writes_audio_attachment(monkeypatch):+ context = FakeContext()+ context.data[CTX_TG_SPEECH_ENABLED] = True+ writes = []++ class FakeRuntime:+ @staticmethod+ def is_globally_enabled():+ return True++ @staticmethod+ async def synthesize_sentences(sentences):+ assert sentences == ["Hello from Agent Zero."]+ return "UklGRg=="++ monkeypatch.setattr(speech.files, "write_file_base64", lambda path, audio: writes.append((path, audio)))+ monkeypatch.setattr(speech.files, "get_abs_path_dockerized", lambda path: f"/a0/{path}")+ monkeypatch.setitem(__import__("sys").modules, "plugins._kokoro_tts.helpers.runtime", FakeRuntime)++ path, warning = asyncio.run(speech.synthesize_attachment(context, "Hello from Agent Zero."))++ assert warning is None+ assert path.startswith("/a0/usr/uploads/telegram_speech_ctx-speech_")+ assert path.endswith(".wav")+ assert writes[0][1] == "UklGRg=="+++def test_final_reply_includes_speech_attachment(monkeypatch):+ agent = FakeAgent()+ sent = []++ async def fake_synthesize(context, text):+ return "/a0/usr/uploads/reply.wav", None++ async def fake_send_reply(context, response_text, attachments, keyboard):+ sent.append(+ {+ "text": response_text,+ "attachments": attachments,+ "keyboard": keyboard,+ }+ )++ monkeypatch.setattr(_55_telegram_reply, "_extract_last_response", lambda context: "Final reply.")+ monkeypatch.setattr(speech, "synthesize_attachment", fake_synthesize)++ extension = _55_telegram_reply.TelegramAutoReply(agent=agent)+ extension._send_reply = fake_send_reply++ asyncio.run(extension.execute())++ assert sent == [+ {+ "text": "Final reply.",+ "attachments": ["/a0/usr/uploads/reply.wav"],+ "keyboard": None,+ }+ ]+ assert CTX_TG_REPLY_TO not in agent.context.data