Skip to content

Agent Zero AI framework

Add Telegram speech mode

Anmol Malik709b86539965
diff --git a/helpers/integration_commands.py b/helpers/integration_commands.py
index e4a09b32b..b555d2af0 100644
--- a/helpers/integration_commands.py
+++ b/helpers/integration_commands.py
@@ -70,6 +70,12 @@ COMMAND_REGISTRY: tuple[IntegrationCommandDef, ...] = (
"Configuration",
args_hint="[on|off]",
),
+ IntegrationCommandDef(
+ "speech",
+ "Enable or disable Telegram spoken replies.",
+ "Configuration",
+ args_hint="[on|off]",
+ ),
IntegrationCommandDef(
"project",
"Show or switch the active project.",
@@ -194,6 +200,8 @@ def try_handle_command(context: "AgentContext", text: str) -> str | None:
return _handle_toggle(context, args, "telegram_stream_enabled", "Response streaming")
if command == "/tools":
return _handle_toggle(context, args, "telegram_tools_enabled", "Tool progress")
+ if command == "/speech":
+ return _handle_toggle(context, args, "telegram_speech_enabled", "Speech replies")
if command == "/project":
return _handle_project(context, args)
if command == "/model":
diff --git a/plugins/_telegram_integration/README.md b/plugins/_telegram_integration/README.md
index a9793fa35..c3c13aae2 100644
--- a/plugins/_telegram_integration/README.md
+++ b/plugins/_telegram_integration/README.md
@@ -24,6 +24,7 @@ This plugin connects one or more Telegram bots to Agent Zero. Each bot runs inde
- `/agent <profile>` switches the active agent profile when the current run is idle.
- `/model`, `/project`, and `/agent` show Telegram inline keyboard pickers when used without arguments.
- `/stream` toggles live response streaming; `/tools` toggles tool progress messages.
+ - `/speech` toggles spoken replies. When enabled, final text replies also include Kokoro-generated Telegram audio when Kokoro TTS is available.
- `/send` or `/queue send` flushes queued messages for the current chat.
- `/steer <message>` sends an intervention to the active run.
- **Group support**
@@ -35,6 +36,7 @@ This plugin connects one or more Telegram bots to Agent Zero. Each bot runs inde
- **Reply delivery**
- Streams tool progress and response text through real Telegram messages updated with `editMessageText`.
- Tool progress and the AI response are separate messages; only the AI response replies to the user message.
+ - Sends native Telegram media: photos, voice/audio files, videos, and document fallback.
- `tool_execute_after` intercepts the `response` tool — sends `break_loop=false` updates as separate intermediate Telegram messages.
- `process_chain_end` auto-sends the final response, with retry logic on failure.
- **Formatting**
@@ -63,6 +65,7 @@ This plugin connects one or more Telegram bots to Agent Zero. Each bot runs inde
- `helpers/telegram_client.py` — Low-level Telegram API wrapper: send text/file/photo, Markdown→HTML converter, keyboard builder, message splitting.
- `helpers/command_ui.py` — Telegram inline keyboard command pickers and callback handling.
- `helpers/draft_stream.py` — Editable Telegram message streaming for tool progress and response previews.
+ - `helpers/speech.py` — Telegram speech-mode TTS synthesis and audio attachment generation.
- **Extensions**
- `extensions/python/job_loop/_10_telegram_bot.py` — Bot lifecycle manager, starts/stops bots on each tick.
- `extensions/python/message_loop_start/_45_telegram_draft_start.py` — Initializes Telegram response streaming.
diff --git a/plugins/_telegram_integration/extensions/python/process_chain_end/_55_telegram_reply.py b/plugins/_telegram_integration/extensions/python/process_chain_end/_55_telegram_reply.py
index 508c3acf4..582d444b2 100644
--- a/plugins/_telegram_integration/extensions/python/process_chain_end/_55_telegram_reply.py
+++ b/plugins/_telegram_integration/extensions/python/process_chain_end/_55_telegram_reply.py
@@ -32,7 +32,17 @@ class TelegramAutoReply(Extension):
try:
if response_text:
+ from plugins._telegram_integration.helpers import speech
+
+ speech_attachment, speech_warning = await speech.synthesize_attachment(
+ context, response_text
+ )
+ if speech_attachment:
+ attachments = [*attachments, speech_attachment]
await self._send_reply(context, response_text, attachments, keyboard)
+ warning = speech.consume_warning(context, speech_warning)
+ if warning:
+ await self._send_reply(context, warning, [], None)
except Exception as e:
PrintStyle.error(f"Telegram auto-reply error: {format_error(e)}")
finally:
diff --git a/plugins/_telegram_integration/helpers/command_ui.py b/plugins/_telegram_integration/helpers/command_ui.py
index a0f108d0f..6ceb3fe6e 100644
--- a/plugins/_telegram_integration/helpers/command_ui.py
+++ b/plugins/_telegram_integration/helpers/command_ui.py
@@ -13,6 +13,7 @@ from plugins._telegram_integration.helpers import telegram_client as tc
from plugins._telegram_integration.helpers.constants import (
CTX_TG_STREAM_ENABLED,
CTX_TG_TOOLS_ENABLED,
+ CTX_TG_SPEECH_ENABLED,
CTX_TG_BOT,
CTX_TG_CHAT_ID,
CTX_TG_CHAT_TYPE,
@@ -85,6 +86,17 @@ async def handle_command(
args,
)
return True
+ if command == "/speech":
+ await send_toggle_picker(
+ context,
+ token,
+ chat_id,
+ reply_to_message_id,
+ CTX_TG_SPEECH_ENABLED,
+ "Speech replies",
+ args,
+ )
+ return True
return False
@@ -129,9 +141,8 @@ async def handle_callback(
selected_context = await _select_session(context, _safe_int(value))
await edit_session_picker(selected_context or context, token, chat_id, message_id, 0, selected=bool(selected_context))
return True
- if kind in {"stream", "tools"} and action in {"on", "off"}:
- key = CTX_TG_STREAM_ENABLED if kind == "stream" else CTX_TG_TOOLS_ENABLED
- label = "Response streaming" if kind == "stream" else "Tool progress"
+ if kind in {"stream", "tools", "speech"} and action in {"on", "off"}:
+ key, label = _toggle_key_label(kind)
context.set_data(key, action == "on")
save_tmp_chat(context)
mark_dirty_for_context(context.id, reason=f"telegram.{kind}_toggle")
@@ -334,7 +345,7 @@ def _agent_view(
def _toggle_view(context: AgentContext, key: str, label: str) -> tuple[str, dict]:
enabled = _toggle_enabled(context, key)
- kind = "stream" if key == CTX_TG_STREAM_ENABLED else "tools"
+ kind = _toggle_kind(key)
state = "enabled" if enabled else "disabled"
text = f"{_html(label)}: <b>{state}</b>"
rows = [[
@@ -542,9 +553,27 @@ def _paged_buttons(
def _toggle_enabled(context: AgentContext, key: str) -> bool:
value = context.get_data(key)
+ if key == CTX_TG_SPEECH_ENABLED:
+ return bool(value)
return True if value is None else bool(value)
+def _toggle_kind(key: str) -> str:
+ if key == CTX_TG_STREAM_ENABLED:
+ return "stream"
+ if key == CTX_TG_TOOLS_ENABLED:
+ return "tools"
+ return "speech"
+
+
+def _toggle_key_label(kind: str) -> tuple[str, str]:
+ if kind == "stream":
+ return CTX_TG_STREAM_ENABLED, "Response streaming"
+ if kind == "tools":
+ return CTX_TG_TOOLS_ENABLED, "Tool progress"
+ return CTX_TG_SPEECH_ENABLED, "Speech replies"
+
+
def _parse_toggle(args: str) -> bool | None:
value = (args or "").strip().lower()
if value in {"on", "enable", "enabled", "yes", "true", "1"}:
diff --git a/plugins/_telegram_integration/helpers/constants.py b/plugins/_telegram_integration/helpers/constants.py
index 190ee7fa4..9696f2661 100644
--- a/plugins/_telegram_integration/helpers/constants.py
+++ b/plugins/_telegram_integration/helpers/constants.py
@@ -13,6 +13,7 @@ CTX_TG_TYPING_STOP = "_telegram_typing_stop"
CTX_TG_REPLY_TO = "_telegram_reply_to_message_id"
CTX_TG_STREAM_ENABLED = "telegram_stream_enabled"
CTX_TG_TOOLS_ENABLED = "telegram_tools_enabled"
+CTX_TG_SPEECH_ENABLED = "telegram_speech_enabled"
# Transient
CTX_TG_ATTACHMENTS = "_telegram_response_attachments"
@@ -25,3 +26,4 @@ CTX_TG_RESPONSE_LAST_UPDATE = "_telegram_response_last_update"
CTX_TG_ERROR_SENT = "_telegram_error_sent"
CTX_TG_HEARTBEAT_TASK = "_telegram_heartbeat_task"
CTX_TG_HEARTBEAT_STOP = "_telegram_heartbeat_stop"
+CTX_TG_SPEECH_WARNING_SENT = "_telegram_speech_warning_sent"
diff --git a/plugins/_telegram_integration/helpers/speech.py b/plugins/_telegram_integration/helpers/speech.py
new file mode 100644
index 000000000..eb74a5e30
--- /dev/null
+++ b/plugins/_telegram_integration/helpers/speech.py
@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import re
+import uuid
+
+from agent import AgentContext
+from helpers import files
+from plugins._telegram_integration.helpers.constants import (
+ CTX_TG_SPEECH_ENABLED,
+ CTX_TG_SPEECH_WARNING_SENT,
+ DOWNLOAD_FOLDER,
+)
+
+
+MAX_SPEECH_CHARS = 1800
+
+
+def is_enabled(context: AgentContext) -> bool:
+ return bool(context.get_data(CTX_TG_SPEECH_ENABLED))
+
+
+async def synthesize_attachment(context: AgentContext, text: str) -> tuple[str | None, str | None]:
+ if not is_enabled(context):
+ return None, None
+
+ speech_text = _speech_text(text)
+ if not speech_text:
+ return None, "Speech mode is enabled, but there was no readable text to speak."
+
+ try:
+ from plugins._kokoro_tts.helpers import runtime
+ except Exception:
+ return None, _setup_message()
+
+ if not runtime.is_globally_enabled():
+ return None, _setup_message()
+
+ try:
+ audio = await runtime.synthesize_sentences([speech_text])
+ except Exception as exc:
+ return None, f"Speech mode is enabled, but text-to-speech failed: {exc}"
+
+ if not audio:
+ return None, "Speech mode is enabled, but text-to-speech returned no audio."
+
+ name = f"telegram_speech_{context.id}_{uuid.uuid4().hex[:10]}.wav"
+ relative_path = f"{DOWNLOAD_FOLDER}/{name}"
+ files.write_file_base64(relative_path, audio)
+ return files.get_abs_path_dockerized(relative_path), None
+
+
+def consume_warning(context: AgentContext, warning: str | None) -> str | None:
+ if not warning:
+ context.data.pop(CTX_TG_SPEECH_WARNING_SENT, None)
+ return None
+ if context.data.get(CTX_TG_SPEECH_WARNING_SENT):
+ return None
+ context.data[CTX_TG_SPEECH_WARNING_SENT] = True
+ return warning
+
+
+def _setup_message() -> str:
+ return (
+ "Speech mode is enabled, but Kokoro TTS is not available. "
+ "Enable the Kokoro TTS plugin from Agent Zero settings, then try again."
+ )
+
+
+def _speech_text(text: str) -> str:
+ value = str(text or "")
+ value = re.sub(r"```.*?```", " ", value, flags=re.DOTALL)
+ value = re.sub(r"`([^`]+)`", r"\1", value)
+ value = re.sub(r"!\[([^\]]*)\]\([^)]+\)", r"\1", value)
+ value = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", value)
+ value = re.sub(r"<[^>]+>", " ", value)
+ value = re.sub(r"[*_#>\-]+", " ", value)
+ value = re.sub(r"\s+", " ", value).strip()
+ if len(value) > MAX_SPEECH_CHARS:
+ value = value[:MAX_SPEECH_CHARS].rstrip()
+ return value
diff --git a/tests/test_telegram_speech_mode.py b/tests/test_telegram_speech_mode.py
new file mode 100644
index 000000000..575cc2777
--- /dev/null
+++ b/tests/test_telegram_speech_mode.py
@@ -0,0 +1,112 @@
+import asyncio
+
+from plugins._telegram_integration.extensions.python.process_chain_end import (
+ _55_telegram_reply,
+)
+from plugins._telegram_integration.helpers import command_ui, speech
+from plugins._telegram_integration.helpers.constants import (
+ CTX_TG_BOT,
+ CTX_TG_REPLY_TO,
+ CTX_TG_SPEECH_ENABLED,
+)
+
+
+class FakeContext:
+ id = "ctx-speech"
+
+ def __init__(self):
+ self.data = {}
+
+ def get_data(self, key):
+ return self.data.get(key)
+
+ def set_data(self, key, value):
+ self.data[key] = value
+
+
+class FakeAgent:
+ number = 0
+
+ def __init__(self):
+ self.context = FakeContext()
+ self.context.data[CTX_TG_BOT] = "main"
+ self.context.data[CTX_TG_REPLY_TO] = 456
+
+
+def test_speech_toggle_defaults_off():
+ context = FakeContext()
+
+ text, markup = command_ui._toggle_view(context, CTX_TG_SPEECH_ENABLED, "Speech replies")
+
+ assert "Speech replies: <b>disabled</b>" == text
+ assert markup["inline_keyboard"][0][0]["callback_data"] == "tg:speech:on"
+ assert markup["inline_keyboard"][0][1]["callback_data"] == "tg:speech:off"
+
+
+def test_speech_text_is_cleaned_for_tts():
+ cleaned = speech._speech_text(
+ "**Hello** [`Agent Zero`](https://example.com)\n\n```python\nprint('skip')\n```"
+ )
+
+ assert cleaned == "Hello Agent Zero"
+
+
+def test_speech_synthesis_writes_audio_attachment(monkeypatch):
+ context = FakeContext()
+ context.data[CTX_TG_SPEECH_ENABLED] = True
+ writes = []
+
+ class FakeRuntime:
+ @staticmethod
+ def is_globally_enabled():
+ return True
+
+ @staticmethod
+ async def synthesize_sentences(sentences):
+ assert sentences == ["Hello from Agent Zero."]
+ return "UklGRg=="
+
+ monkeypatch.setattr(speech.files, "write_file_base64", lambda path, audio: writes.append((path, audio)))
+ monkeypatch.setattr(speech.files, "get_abs_path_dockerized", lambda path: f"/a0/{path}")
+ monkeypatch.setitem(__import__("sys").modules, "plugins._kokoro_tts.helpers.runtime", FakeRuntime)
+
+ path, warning = asyncio.run(speech.synthesize_attachment(context, "Hello from Agent Zero."))
+
+ assert warning is None
+ assert path.startswith("/a0/usr/uploads/telegram_speech_ctx-speech_")
+ assert path.endswith(".wav")
+ assert writes[0][1] == "UklGRg=="
+
+
+def test_final_reply_includes_speech_attachment(monkeypatch):
+ agent = FakeAgent()
+ sent = []
+
+ async def fake_synthesize(context, text):
+ return "/a0/usr/uploads/reply.wav", None
+
+ async def fake_send_reply(context, response_text, attachments, keyboard):
+ sent.append(
+ {
+ "text": response_text,
+ "attachments": attachments,
+ "keyboard": keyboard,
+ }
+ )
+
+ monkeypatch.setattr(_55_telegram_reply, "_extract_last_response", lambda context: "Final reply.")
+ monkeypatch.setattr(speech, "synthesize_attachment", fake_synthesize)
+
+ extension = _55_telegram_reply.TelegramAutoReply(agent=agent)
+ extension._send_reply = fake_send_reply
+
+ asyncio.run(extension.execute())
+
+ assert sent == [
+ {
+ "text": "Final reply.",
+ "attachments": ["/a0/usr/uploads/reply.wav"],
+ "keyboard": None,
+ }
+ ]
+ assert CTX_TG_REPLY_TO not in agent.context.data