The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
refactor(gateway): move inbound voice transcription into its own mixin
Teknium3edbc6f45964
diff --git a/gateway/run_inbound.py b/gateway/run_inbound.pyindex 7347c4e78e3..9f2accb7051 100644--- a/gateway/run_inbound.py+++ b/gateway/run_inbound.py@@ -27,6 +27,7 @@ from gateway.run_busy import approval_input_wordsfrom gateway.run_common import _UNSETfrom gateway.run_inbound_media import rehome_inbound_mediafrom gateway.run_inbound_hooks import GatewayInboundHooksMixin+from gateway.run_inbound_transcription import GatewayInboundTranscriptionMixinfrom gateway.run_plugin_injection import GatewayPluginInjectionMixinfrom gateway.run_inbound_unauthorized import (UnauthorizedOwnerNotifier, pairing_code_reply, pairing_profile_arg, pairing_rate_limited_reply,@@ -68,7 +69,7 @@ def strip_discord_triggering_note(event: Any, message_text: Any) -> Any:return message_text.removeprefix(prefix)-class GatewayInboundMixin(GatewayInboundHooksMixin, GatewayPluginInjectionMixin):+class GatewayInboundMixin(GatewayInboundHooksMixin, GatewayInboundTranscriptionMixin, GatewayPluginInjectionMixin):"""Inbound message pipeline (_handle_message, text/media preparation, durable-turn markers, plugin injection) for GatewayRunner."""async def _hm_offer_pairing_code(self, source: SessionSource) -> None:@@ -1877,130 +1878,3 @@ class GatewayInboundMixin(GatewayInboundHooksMixin, GatewayPluginInjectionMixin)# Plain quoted line: a "The user sent a voice message..." wrapper read as a meta-instruction# and made the LLM comment on voice mode instead.return transcript, f'"{transcript}"'-- async def _enrich_message_with_transcription(- self, user_text: str, audio_paths: list[str], clip_memo: Optional[dict] = None- ) -> tuple[str, list[str]]:- """Transcribe voice clips with the configured STT provider and prepend the transcripts →- ``(enriched_text, successful_transcripts)``; the transcripts (input order; empty if every clip- failed or STT is disabled) let callers echo them back before the agent loop. ``clip_memo``- (path → ``_transcribe_one_clip`` result) skips clips an earlier pass already transcribed."""- from gateway.run import _probe_audio_duration- audio_paths = list(dict.fromkeys(audio_paths))- if not getattr(self.config, "stt_enabled", True):- notes = []- for path in audio_paths:- abs_path = os.path.abspath(path)- duration_str = await _probe_audio_duration(abs_path)- suffix = f" (duration: {duration_str})" if duration_str else ""- notes.append(f"[The user sent a voice message: {abs_path}{suffix}]")- return (self._prepend_media_prefix("\n\n".join(notes), user_text) if notes else user_text), []-- try:- from tools.transcription_tools import (- transcribe_audio, transcribe_audio_local_fallback- )- except ModuleNotFoundError as e:- logger.error("Transcription module unavailable: %s", e)- return self._prepend_media_prefix("[voice message could not be transcribed]", user_text), []-- enriched_parts = []- successful_transcripts: list[str] = []- for path in audio_paths:- try:- if clip_memo is not None and path in clip_memo:- transcript, note = clip_memo[path]- else:- logger.debug("Transcribing user voice: %s", path)- transcript, note = await self._transcribe_one_clip(- path, transcribe_audio, transcribe_audio_local_fallback,- )- if clip_memo is not None:- clip_memo[path] = (transcript, note)- if transcript is not None:- successful_transcripts.append(transcript)- enriched_parts.append(note)- except Exception as e:- logger.error("Transcription error: %s", e)- enriched_parts.append(self._untranscribed_audio_note(path))-- if enriched_parts:- user_text = self._prepend_media_prefix("\n\n".join(enriched_parts), user_text)- return user_text, successful_transcripts-- def _pending_event_audio_paths(self, event) -> list[str]:- """Return STT-eligible paths from a pending voice message."""- from gateway.run import _event_media_is_stt_input- return [- path for i, path in enumerate(getattr(event, "media_urls", None) or [])- if _event_media_is_stt_input(event, i)- ]-- async def _transcribe_pending_audio_event_once(- self, event, user_text: Optional[str] = None- ) -> tuple[str | None, list[str]]:- """Transcribe a pending audio event once and cache the result on the event: the interrupt- monitor and the pending-drain path both need it — one STT call and one echo per message.- ``user_text`` is a deliberate override of the caption; ``None`` reads ``event.text`` live."""- if hasattr(event, "_gateway_pending_stt_text"):- return event._gateway_pending_stt_text, list(getattr(event, "_gateway_pending_stt_transcripts", []) or [])- audio_paths = self._pending_event_audio_paths(event)- if not audio_paths:- return user_text if user_text is not None else (getattr(event, "text", None) or None), []- text = user_text if user_text is not None else (getattr(event, "text", "") or "")- # Per-clip results outlive the merged-event cache below (a merge drops only that), so every- # re-run — this loop's or the drain path's — transcribes just the newly merged clips.- clip_memo = getattr(event, "_gateway_pending_stt_clips", None)- if clip_memo is None:- clip_memo = {}- event._gateway_pending_stt_clips = clip_memo- enriched_text, successful_transcripts = await self._enrich_message_with_transcription(text, audio_paths, clip_memo)- # A voice note merged in during the await dropped a cache that did not exist yet; re-run on- # the merged media rather than publish a transcript that silently omits the new note.- while self._pending_event_audio_paths(event) != audio_paths:- audio_paths = self._pending_event_audio_paths(event)- text = user_text if user_text is not None else (getattr(event, "text", "") or "")- enriched_text, successful_transcripts = await self._enrich_message_with_transcription(- text, audio_paths, clip_memo,- )- event._gateway_pending_stt_text = enriched_text- event._gateway_pending_stt_transcripts = list(successful_transcripts)- return enriched_text, successful_transcripts-- async def _echo_pending_stt_transcripts_once(- self, event, adapter, source, transcripts: list[str], *, metadata=None,- log_context: str = "Transcript",- ) -> None:- """Echo pending-event STT transcripts to the chat at most once. Tracked as a COUNT (not a- set — identical transcripts are distinct deliveries): ``merge_pending_message_event`` can- append a second voice note and invalidate the cache; the re-run returns earlier transcripts- as a prefix, so only the unsent tail is echoed."""- if not transcripts or not self._should_echo_stt_transcripts() or adapter is None:- return- already_echoed = int(getattr(event, "_gateway_pending_stt_echoed", 0) or 0)- event._gateway_pending_stt_echoed = max(already_echoed, len(transcripts))- await self._echo_stt_transcripts(- adapter, source, transcripts[already_echoed:], metadata=metadata, log_context=log_context,- )-- async def _transcribe_and_echo_pending_voice(- self, event, adapter, source, text: str, *, log_context: str, metadata=_UNSET- ) -> tuple[str, list[str]]:- """Transcribe a pending voice event and echo transcripts once → ``(enriched_text,- transcripts)`` for ``agent.interrupt()`` or the pending-drain flow; ``(text, [])`` when there- is no STT-eligible media (caller owns the ``_build_media_placeholder`` fallback). ``text`` is- only that fallback: the caption is read live from ``event.text`` so one merged in during- the STT await is kept."""- if not self._pending_event_audio_paths(event):- return text, []- try:- enriched_text, transcripts = await self._transcribe_pending_audio_event_once(event)- if metadata is _UNSET:- metadata = self._thread_metadata_for_source(source, self._reply_anchor_for_event(event))- await self._echo_pending_stt_transcripts_once(- event, adapter, source, transcripts, metadata=metadata, log_context=log_context- )- return enriched_text or text, transcripts- except Exception as trans_exc:- logger.warning("%s transcription failed: %s", log_context, trans_exc)- return text, []diff --git a/gateway/run_inbound_transcription.py b/gateway/run_inbound_transcription.pynew file mode 100644index 00000000000..b29fc18fd1a--- /dev/null+++ b/gateway/run_inbound_transcription.py@@ -0,0 +1,143 @@+"""Voice-clip transcription for inbound messages (moved out of ``gateway/run_inbound.py``)."""++from __future__ import annotations++import logging+import os+from typing import Optional++from gateway.run_common import _UNSET++# Log-record parity with the origin module.+logger = logging.getLogger("gateway.run")+++class GatewayInboundTranscriptionMixin:+ """STT enrichment and the once-only echo of pending voice transcripts."""++ async def _enrich_message_with_transcription(+ self, user_text: str, audio_paths: list[str], clip_memo: Optional[dict] = None+ ) -> tuple[str, list[str]]:+ """Transcribe voice clips with the configured STT provider and prepend the transcripts →+ ``(enriched_text, successful_transcripts)``; the transcripts (input order; empty if every clip+ failed or STT is disabled) let callers echo them back before the agent loop. ``clip_memo``+ (path → ``_transcribe_one_clip`` result) skips clips an earlier pass already transcribed."""+ from gateway.run import _probe_audio_duration+ audio_paths = list(dict.fromkeys(audio_paths))+ if not getattr(self.config, "stt_enabled", True):+ notes = []+ for path in audio_paths:+ abs_path = os.path.abspath(path)+ duration_str = await _probe_audio_duration(abs_path)+ suffix = f" (duration: {duration_str})" if duration_str else ""+ notes.append(f"[The user sent a voice message: {abs_path}{suffix}]")+ return (self._prepend_media_prefix("\n\n".join(notes), user_text) if notes else user_text), []++ try:+ from tools.transcription_tools import (+ transcribe_audio, transcribe_audio_local_fallback+ )+ except ModuleNotFoundError as e:+ logger.error("Transcription module unavailable: %s", e)+ return self._prepend_media_prefix("[voice message could not be transcribed]", user_text), []++ enriched_parts = []+ successful_transcripts: list[str] = []+ for path in audio_paths:+ try:+ if clip_memo is not None and path in clip_memo:+ transcript, note = clip_memo[path]+ else:+ logger.debug("Transcribing user voice: %s", path)+ transcript, note = await self._transcribe_one_clip(+ path, transcribe_audio, transcribe_audio_local_fallback,+ )+ if clip_memo is not None:+ clip_memo[path] = (transcript, note)+ if transcript is not None:+ successful_transcripts.append(transcript)+ enriched_parts.append(note)+ except Exception as e:+ logger.error("Transcription error: %s", e)+ enriched_parts.append(self._untranscribed_audio_note(path))++ if enriched_parts:+ user_text = self._prepend_media_prefix("\n\n".join(enriched_parts), user_text)+ return user_text, successful_transcripts++ def _pending_event_audio_paths(self, event) -> list[str]:+ """Return STT-eligible paths from a pending voice message."""+ from gateway.run import _event_media_is_stt_input+ return [+ path for i, path in enumerate(getattr(event, "media_urls", None) or [])+ if _event_media_is_stt_input(event, i)+ ]++ async def _transcribe_pending_audio_event_once(+ self, event, user_text: Optional[str] = None+ ) -> tuple[str | None, list[str]]:+ """Transcribe a pending audio event once and cache the result on the event: the interrupt+ monitor and the pending-drain path both need it — one STT call and one echo per message.+ ``user_text`` is a deliberate override of the caption; ``None`` reads ``event.text`` live."""+ if hasattr(event, "_gateway_pending_stt_text"):+ return event._gateway_pending_stt_text, list(getattr(event, "_gateway_pending_stt_transcripts", []) or [])+ audio_paths = self._pending_event_audio_paths(event)+ if not audio_paths:+ return user_text if user_text is not None else (getattr(event, "text", None) or None), []+ text = user_text if user_text is not None else (getattr(event, "text", "") or "")+ # Per-clip results outlive the merged-event cache below (a merge drops only that), so every+ # re-run — this loop's or the drain path's — transcribes just the newly merged clips.+ clip_memo = getattr(event, "_gateway_pending_stt_clips", None)+ if clip_memo is None:+ clip_memo = {}+ event._gateway_pending_stt_clips = clip_memo+ enriched_text, successful_transcripts = await self._enrich_message_with_transcription(text, audio_paths, clip_memo)+ # A voice note merged in during the await dropped a cache that did not exist yet; re-run on+ # the merged media rather than publish a transcript that silently omits the new note.+ while self._pending_event_audio_paths(event) != audio_paths:+ audio_paths = self._pending_event_audio_paths(event)+ text = user_text if user_text is not None else (getattr(event, "text", "") or "")+ enriched_text, successful_transcripts = await self._enrich_message_with_transcription(+ text, audio_paths, clip_memo,+ )+ event._gateway_pending_stt_text = enriched_text+ event._gateway_pending_stt_transcripts = list(successful_transcripts)+ return enriched_text, successful_transcripts++ async def _echo_pending_stt_transcripts_once(+ self, event, adapter, source, transcripts: list[str], *, metadata=None,+ log_context: str = "Transcript",+ ) -> None:+ """Echo pending-event STT transcripts to the chat at most once. Tracked as a COUNT (not a+ set — identical transcripts are distinct deliveries): ``merge_pending_message_event`` can+ append a second voice note and invalidate the cache; the re-run returns earlier transcripts+ as a prefix, so only the unsent tail is echoed."""+ if not transcripts or not self._should_echo_stt_transcripts() or adapter is None:+ return+ already_echoed = int(getattr(event, "_gateway_pending_stt_echoed", 0) or 0)+ event._gateway_pending_stt_echoed = max(already_echoed, len(transcripts))+ await self._echo_stt_transcripts(+ adapter, source, transcripts[already_echoed:], metadata=metadata, log_context=log_context,+ )++ async def _transcribe_and_echo_pending_voice(+ self, event, adapter, source, text: str, *, log_context: str, metadata=_UNSET+ ) -> tuple[str, list[str]]:+ """Transcribe a pending voice event and echo transcripts once → ``(enriched_text,+ transcripts)`` for ``agent.interrupt()`` or the pending-drain flow; ``(text, [])`` when there+ is no STT-eligible media (caller owns the ``_build_media_placeholder`` fallback). ``text`` is+ only that fallback: the caption is read live from ``event.text`` so one merged in during+ the STT await is kept."""+ if not self._pending_event_audio_paths(event):+ return text, []+ try:+ enriched_text, transcripts = await self._transcribe_pending_audio_event_once(event)+ if metadata is _UNSET:+ metadata = self._thread_metadata_for_source(source, self._reply_anchor_for_event(event))+ await self._echo_pending_stt_transcripts_once(+ event, adapter, source, transcripts, metadata=metadata, log_context=log_context+ )+ return enriched_text or text, transcripts+ except Exception as trans_exc:+ logger.warning("%s transcription failed: %s", log_context, trans_exc)+ return text, []