Skip to content

The agent that grows with you

refactor(gateway): move inbound voice transcription into its own mixin

Teknium3edbc6f45964
diff --git a/gateway/run_inbound.py b/gateway/run_inbound.py
index 7347c4e78e3..9f2accb7051 100644
--- a/gateway/run_inbound.py
+++ b/gateway/run_inbound.py
@@ -27,6 +27,7 @@ from gateway.run_busy import approval_input_words
from gateway.run_common import _UNSET
from gateway.run_inbound_media import rehome_inbound_media
from gateway.run_inbound_hooks import GatewayInboundHooksMixin
+from gateway.run_inbound_transcription import GatewayInboundTranscriptionMixin
from gateway.run_plugin_injection import GatewayPluginInjectionMixin
from gateway.run_inbound_unauthorized import (
UnauthorizedOwnerNotifier, pairing_code_reply, pairing_profile_arg, pairing_rate_limited_reply,
@@ -68,7 +69,7 @@ def strip_discord_triggering_note(event: Any, message_text: Any) -> Any:
return message_text.removeprefix(prefix)
-class GatewayInboundMixin(GatewayInboundHooksMixin, GatewayPluginInjectionMixin):
+class GatewayInboundMixin(GatewayInboundHooksMixin, GatewayInboundTranscriptionMixin, GatewayPluginInjectionMixin):
"""Inbound message pipeline (_handle_message, text/media preparation, durable-turn markers, plugin injection) for GatewayRunner."""
async def _hm_offer_pairing_code(self, source: SessionSource) -> None:
@@ -1877,130 +1878,3 @@ class GatewayInboundMixin(GatewayInboundHooksMixin, GatewayPluginInjectionMixin)
# Plain quoted line: a "The user sent a voice message..." wrapper read as a meta-instruction
# and made the LLM comment on voice mode instead.
return transcript, f'"{transcript}"'
-
- async def _enrich_message_with_transcription(
- self, user_text: str, audio_paths: list[str], clip_memo: Optional[dict] = None
- ) -> tuple[str, list[str]]:
- """Transcribe voice clips with the configured STT provider and prepend the transcripts →
- ``(enriched_text, successful_transcripts)``; the transcripts (input order; empty if every clip
- failed or STT is disabled) let callers echo them back before the agent loop. ``clip_memo``
- (path → ``_transcribe_one_clip`` result) skips clips an earlier pass already transcribed."""
- from gateway.run import _probe_audio_duration
- audio_paths = list(dict.fromkeys(audio_paths))
- if not getattr(self.config, "stt_enabled", True):
- notes = []
- for path in audio_paths:
- abs_path = os.path.abspath(path)
- duration_str = await _probe_audio_duration(abs_path)
- suffix = f" (duration: {duration_str})" if duration_str else ""
- notes.append(f"[The user sent a voice message: {abs_path}{suffix}]")
- return (self._prepend_media_prefix("\n\n".join(notes), user_text) if notes else user_text), []
-
- try:
- from tools.transcription_tools import (
- transcribe_audio, transcribe_audio_local_fallback
- )
- except ModuleNotFoundError as e:
- logger.error("Transcription module unavailable: %s", e)
- return self._prepend_media_prefix("[voice message could not be transcribed]", user_text), []
-
- enriched_parts = []
- successful_transcripts: list[str] = []
- for path in audio_paths:
- try:
- if clip_memo is not None and path in clip_memo:
- transcript, note = clip_memo[path]
- else:
- logger.debug("Transcribing user voice: %s", path)
- transcript, note = await self._transcribe_one_clip(
- path, transcribe_audio, transcribe_audio_local_fallback,
- )
- if clip_memo is not None:
- clip_memo[path] = (transcript, note)
- if transcript is not None:
- successful_transcripts.append(transcript)
- enriched_parts.append(note)
- except Exception as e:
- logger.error("Transcription error: %s", e)
- enriched_parts.append(self._untranscribed_audio_note(path))
-
- if enriched_parts:
- user_text = self._prepend_media_prefix("\n\n".join(enriched_parts), user_text)
- return user_text, successful_transcripts
-
- def _pending_event_audio_paths(self, event) -> list[str]:
- """Return STT-eligible paths from a pending voice message."""
- from gateway.run import _event_media_is_stt_input
- return [
- path for i, path in enumerate(getattr(event, "media_urls", None) or [])
- if _event_media_is_stt_input(event, i)
- ]
-
- async def _transcribe_pending_audio_event_once(
- self, event, user_text: Optional[str] = None
- ) -> tuple[str | None, list[str]]:
- """Transcribe a pending audio event once and cache the result on the event: the interrupt
- monitor and the pending-drain path both need it — one STT call and one echo per message.
- ``user_text`` is a deliberate override of the caption; ``None`` reads ``event.text`` live."""
- if hasattr(event, "_gateway_pending_stt_text"):
- return event._gateway_pending_stt_text, list(getattr(event, "_gateway_pending_stt_transcripts", []) or [])
- audio_paths = self._pending_event_audio_paths(event)
- if not audio_paths:
- return user_text if user_text is not None else (getattr(event, "text", None) or None), []
- text = user_text if user_text is not None else (getattr(event, "text", "") or "")
- # Per-clip results outlive the merged-event cache below (a merge drops only that), so every
- # re-run — this loop's or the drain path's — transcribes just the newly merged clips.
- clip_memo = getattr(event, "_gateway_pending_stt_clips", None)
- if clip_memo is None:
- clip_memo = {}
- event._gateway_pending_stt_clips = clip_memo
- enriched_text, successful_transcripts = await self._enrich_message_with_transcription(text, audio_paths, clip_memo)
- # A voice note merged in during the await dropped a cache that did not exist yet; re-run on
- # the merged media rather than publish a transcript that silently omits the new note.
- while self._pending_event_audio_paths(event) != audio_paths:
- audio_paths = self._pending_event_audio_paths(event)
- text = user_text if user_text is not None else (getattr(event, "text", "") or "")
- enriched_text, successful_transcripts = await self._enrich_message_with_transcription(
- text, audio_paths, clip_memo,
- )
- event._gateway_pending_stt_text = enriched_text
- event._gateway_pending_stt_transcripts = list(successful_transcripts)
- return enriched_text, successful_transcripts
-
- async def _echo_pending_stt_transcripts_once(
- self, event, adapter, source, transcripts: list[str], *, metadata=None,
- log_context: str = "Transcript",
- ) -> None:
- """Echo pending-event STT transcripts to the chat at most once. Tracked as a COUNT (not a
- set — identical transcripts are distinct deliveries): ``merge_pending_message_event`` can
- append a second voice note and invalidate the cache; the re-run returns earlier transcripts
- as a prefix, so only the unsent tail is echoed."""
- if not transcripts or not self._should_echo_stt_transcripts() or adapter is None:
- return
- already_echoed = int(getattr(event, "_gateway_pending_stt_echoed", 0) or 0)
- event._gateway_pending_stt_echoed = max(already_echoed, len(transcripts))
- await self._echo_stt_transcripts(
- adapter, source, transcripts[already_echoed:], metadata=metadata, log_context=log_context,
- )
-
- async def _transcribe_and_echo_pending_voice(
- self, event, adapter, source, text: str, *, log_context: str, metadata=_UNSET
- ) -> tuple[str, list[str]]:
- """Transcribe a pending voice event and echo transcripts once → ``(enriched_text,
- transcripts)`` for ``agent.interrupt()`` or the pending-drain flow; ``(text, [])`` when there
- is no STT-eligible media (caller owns the ``_build_media_placeholder`` fallback). ``text`` is
- only that fallback: the caption is read live from ``event.text`` so one merged in during
- the STT await is kept."""
- if not self._pending_event_audio_paths(event):
- return text, []
- try:
- enriched_text, transcripts = await self._transcribe_pending_audio_event_once(event)
- if metadata is _UNSET:
- metadata = self._thread_metadata_for_source(source, self._reply_anchor_for_event(event))
- await self._echo_pending_stt_transcripts_once(
- event, adapter, source, transcripts, metadata=metadata, log_context=log_context
- )
- return enriched_text or text, transcripts
- except Exception as trans_exc:
- logger.warning("%s transcription failed: %s", log_context, trans_exc)
- return text, []
diff --git a/gateway/run_inbound_transcription.py b/gateway/run_inbound_transcription.py
new file mode 100644
index 00000000000..b29fc18fd1a
--- /dev/null
+++ b/gateway/run_inbound_transcription.py
@@ -0,0 +1,143 @@
+"""Voice-clip transcription for inbound messages (moved out of ``gateway/run_inbound.py``)."""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Optional
+
+from gateway.run_common import _UNSET
+
+# Log-record parity with the origin module.
+logger = logging.getLogger("gateway.run")
+
+
+class GatewayInboundTranscriptionMixin:
+ """STT enrichment and the once-only echo of pending voice transcripts."""
+
+ async def _enrich_message_with_transcription(
+ self, user_text: str, audio_paths: list[str], clip_memo: Optional[dict] = None
+ ) -> tuple[str, list[str]]:
+ """Transcribe voice clips with the configured STT provider and prepend the transcripts →
+ ``(enriched_text, successful_transcripts)``; the transcripts (input order; empty if every clip
+ failed or STT is disabled) let callers echo them back before the agent loop. ``clip_memo``
+ (path → ``_transcribe_one_clip`` result) skips clips an earlier pass already transcribed."""
+ from gateway.run import _probe_audio_duration
+ audio_paths = list(dict.fromkeys(audio_paths))
+ if not getattr(self.config, "stt_enabled", True):
+ notes = []
+ for path in audio_paths:
+ abs_path = os.path.abspath(path)
+ duration_str = await _probe_audio_duration(abs_path)
+ suffix = f" (duration: {duration_str})" if duration_str else ""
+ notes.append(f"[The user sent a voice message: {abs_path}{suffix}]")
+ return (self._prepend_media_prefix("\n\n".join(notes), user_text) if notes else user_text), []
+
+ try:
+ from tools.transcription_tools import (
+ transcribe_audio, transcribe_audio_local_fallback
+ )
+ except ModuleNotFoundError as e:
+ logger.error("Transcription module unavailable: %s", e)
+ return self._prepend_media_prefix("[voice message could not be transcribed]", user_text), []
+
+ enriched_parts = []
+ successful_transcripts: list[str] = []
+ for path in audio_paths:
+ try:
+ if clip_memo is not None and path in clip_memo:
+ transcript, note = clip_memo[path]
+ else:
+ logger.debug("Transcribing user voice: %s", path)
+ transcript, note = await self._transcribe_one_clip(
+ path, transcribe_audio, transcribe_audio_local_fallback,
+ )
+ if clip_memo is not None:
+ clip_memo[path] = (transcript, note)
+ if transcript is not None:
+ successful_transcripts.append(transcript)
+ enriched_parts.append(note)
+ except Exception as e:
+ logger.error("Transcription error: %s", e)
+ enriched_parts.append(self._untranscribed_audio_note(path))
+
+ if enriched_parts:
+ user_text = self._prepend_media_prefix("\n\n".join(enriched_parts), user_text)
+ return user_text, successful_transcripts
+
+ def _pending_event_audio_paths(self, event) -> list[str]:
+ """Return STT-eligible paths from a pending voice message."""
+ from gateway.run import _event_media_is_stt_input
+ return [
+ path for i, path in enumerate(getattr(event, "media_urls", None) or [])
+ if _event_media_is_stt_input(event, i)
+ ]
+
+ async def _transcribe_pending_audio_event_once(
+ self, event, user_text: Optional[str] = None
+ ) -> tuple[str | None, list[str]]:
+ """Transcribe a pending audio event once and cache the result on the event: the interrupt
+ monitor and the pending-drain path both need it — one STT call and one echo per message.
+ ``user_text`` is a deliberate override of the caption; ``None`` reads ``event.text`` live."""
+ if hasattr(event, "_gateway_pending_stt_text"):
+ return event._gateway_pending_stt_text, list(getattr(event, "_gateway_pending_stt_transcripts", []) or [])
+ audio_paths = self._pending_event_audio_paths(event)
+ if not audio_paths:
+ return user_text if user_text is not None else (getattr(event, "text", None) or None), []
+ text = user_text if user_text is not None else (getattr(event, "text", "") or "")
+ # Per-clip results outlive the merged-event cache below (a merge drops only that), so every
+ # re-run — this loop's or the drain path's — transcribes just the newly merged clips.
+ clip_memo = getattr(event, "_gateway_pending_stt_clips", None)
+ if clip_memo is None:
+ clip_memo = {}
+ event._gateway_pending_stt_clips = clip_memo
+ enriched_text, successful_transcripts = await self._enrich_message_with_transcription(text, audio_paths, clip_memo)
+ # A voice note merged in during the await dropped a cache that did not exist yet; re-run on
+ # the merged media rather than publish a transcript that silently omits the new note.
+ while self._pending_event_audio_paths(event) != audio_paths:
+ audio_paths = self._pending_event_audio_paths(event)
+ text = user_text if user_text is not None else (getattr(event, "text", "") or "")
+ enriched_text, successful_transcripts = await self._enrich_message_with_transcription(
+ text, audio_paths, clip_memo,
+ )
+ event._gateway_pending_stt_text = enriched_text
+ event._gateway_pending_stt_transcripts = list(successful_transcripts)
+ return enriched_text, successful_transcripts
+
+ async def _echo_pending_stt_transcripts_once(
+ self, event, adapter, source, transcripts: list[str], *, metadata=None,
+ log_context: str = "Transcript",
+ ) -> None:
+ """Echo pending-event STT transcripts to the chat at most once. Tracked as a COUNT (not a
+ set — identical transcripts are distinct deliveries): ``merge_pending_message_event`` can
+ append a second voice note and invalidate the cache; the re-run returns earlier transcripts
+ as a prefix, so only the unsent tail is echoed."""
+ if not transcripts or not self._should_echo_stt_transcripts() or adapter is None:
+ return
+ already_echoed = int(getattr(event, "_gateway_pending_stt_echoed", 0) or 0)
+ event._gateway_pending_stt_echoed = max(already_echoed, len(transcripts))
+ await self._echo_stt_transcripts(
+ adapter, source, transcripts[already_echoed:], metadata=metadata, log_context=log_context,
+ )
+
+ async def _transcribe_and_echo_pending_voice(
+ self, event, adapter, source, text: str, *, log_context: str, metadata=_UNSET
+ ) -> tuple[str, list[str]]:
+ """Transcribe a pending voice event and echo transcripts once → ``(enriched_text,
+ transcripts)`` for ``agent.interrupt()`` or the pending-drain flow; ``(text, [])`` when there
+ is no STT-eligible media (caller owns the ``_build_media_placeholder`` fallback). ``text`` is
+ only that fallback: the caption is read live from ``event.text`` so one merged in during
+ the STT await is kept."""
+ if not self._pending_event_audio_paths(event):
+ return text, []
+ try:
+ enriched_text, transcripts = await self._transcribe_pending_audio_event_once(event)
+ if metadata is _UNSET:
+ metadata = self._thread_metadata_for_source(source, self._reply_anchor_for_event(event))
+ await self._echo_pending_stt_transcripts_once(
+ event, adapter, source, transcripts, metadata=metadata, log_context=log_context
+ )
+ return enriched_text or text, transcripts
+ except Exception as trans_exc:
+ logger.warning("%s transcription failed: %s", log_context, trans_exc)
+ return text, []