Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
import array
import asyncio
import logging
import os
import uuid
import warnings
from collections import Counter, deque
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
import httpx
import numpy as np
from dotenv import load_dotenv
from livekit import rtc
from livekit.agents import (
Agent,
AgentSession,
AutoSubscribe,
JobContext,
JobProcess,
StopResponse,
cli,
inference,
llm,
room_io,
stt as stt_pkg,
utils,
WorkerOptions,
)
from livekit.plugins import silero
# `resemblyzer` imports `webrtcvad`, which currently emits a noisy setuptools
# `pkg_resources` deprecation warning once per worker process. It is not
# actionable for the agent, so keep production logs focused on runtime issues.
warnings.filterwarnings(
"ignore",
message=r"pkg_resources is deprecated as an API.*",
category=UserWarning,
module=r"webrtcvad",
)
from resemblyzer import VoiceEncoder, preprocess_wav
load_dotenv()
logger = logging.getLogger("macro-transcriber")
MACRO_API_URL = os.environ.get("MACRO_API_URL", "http://localhost:8080")
INTERNAL_CALL_SECRET = os.environ.get("INTERNAL_CALL_SECRET", "")
# Cosine distance threshold for in-call Resemblyzer speaker clustering.
# Lower = stricter / more new diarized ids; higher = looser / more reuse.
VOICE_CLUSTER_DISTANCE_THRESHOLD = 0.30
# Keep this configurable because short utterances are common in calls, but
# Resemblyzer embeddings become noisy when there is too little speech.
VOICE_EMBEDDING_MIN_SECONDS = 1.0
VOICE_EMBEDDING_BUFFER_SECONDS = 3.0
@dataclass
class _VoiceCluster:
diarized_speaker_id: str
centroid: np.ndarray
samples: int = 1
class VoiceClusterResolver:
"""Turn-level speaker clustering based on Resemblyzer embeddings.
Deepgram's live diarization labels can churn between finalized utterances.
This resolver keeps a per-track in-memory set of voice centroids and returns
the same diarized speaker id for turns whose embeddings are close enough.
"""
def __init__(
self,
*,
channel_id: str,
participant_identity: str,
distance_threshold: float = VOICE_CLUSTER_DISTANCE_THRESHOLD,
):
self.channel_id = channel_id
self.participant_identity = participant_identity
self.distance_threshold = distance_threshold
self._namespace = uuid.uuid4().hex
self._clusters: list[_VoiceCluster] = []
def resolve(self, embedding: list[float]) -> tuple[str, float | None, bool] | None:
"""Return (diarized_speaker_id, nearest_distance, created_new)."""
vector = self._normalize(embedding)
if vector is None:
return None
best_index: int | None = None
best_distance: float | None = None
for idx, cluster in enumerate(self._clusters):
distance = self._cosine_distance(cluster.centroid, vector)
if best_distance is None or distance < best_distance:
best_index = idx
best_distance = distance
if (
best_index is not None
and best_distance is not None
and best_distance <= self.distance_threshold
):
cluster = self._clusters[best_index]
self._update_centroid(cluster, vector)
return cluster.diarized_speaker_id, best_distance, False
cluster = _VoiceCluster(
diarized_speaker_id=self._new_cluster_id(len(self._clusters)),
centroid=vector,
)
self._clusters.append(cluster)
return cluster.diarized_speaker_id, best_distance, True
def _new_cluster_id(self, index: int) -> str:
seed = (
f"{self.channel_id}:{self.participant_identity}:"
f"{self._namespace}:voice-cluster:{index}"
)
return str(uuid.uuid5(uuid.NAMESPACE_URL, seed))
@staticmethod
def _normalize(embedding: list[float]) -> np.ndarray | None:
vector = np.asarray(embedding, dtype=np.float32)
if vector.ndim != 1 or vector.size == 0 or not np.isfinite(vector).all():
return None
norm = float(np.linalg.norm(vector))
if norm <= 0.0:
return None
return vector / norm
@staticmethod
def _cosine_distance(a: np.ndarray, b: np.ndarray) -> float:
distance = 1.0 - float(np.dot(a, b))
return max(0.0, min(2.0, distance))
@staticmethod
def _update_centroid(cluster: _VoiceCluster, vector: np.ndarray) -> None:
updated = (cluster.centroid * cluster.samples) + vector
norm = float(np.linalg.norm(updated))
if norm > 0.0:
cluster.centroid = updated / norm
cluster.samples += 1
class Transcriber(Agent):
"""STT-only agent bound to a single participant."""
def __init__(
self,
*,
participant_identity: str,
channel_id: str,
http_client: httpx.AsyncClient,
voice_encoder: VoiceEncoder | None = None,
):
super().__init__(
instructions="Transcribe user speech.",
stt=inference.STT(
"deepgram/nova-3",
language="multi",
extra_kwargs={
# ms of silence before Deepgram finalizes an utterance.
# Library default is 25ms, which cuts hesitant speakers mid-thought.
"endpointing": 400,
"smart_format": False,
"punctuate": True,
"filler_words": True,
"numerals": True,
"interim_results": True,
"no_delay": True,
# Keep provider speaker labels as a fallback for very short
# turns where Resemblyzer cannot produce a stable embedding.
"diarize": True,
},
),
)
self.participant_identity = participant_identity
self.channel_id = channel_id
self.http_client = http_client
self.voice_encoder = voice_encoder
# Rolling buffer of recent audio frames keyed off this participant's
# stream. Used to compute a speaker-embedding on turn completion.
# `_audio_sample_rate` and `_audio_num_channels` are captured from
# the first frame so we can stitch the buffer back into a flat PCM
# array at embed time without re-inspecting every frame.
self._audio_frames: deque[rtc.AudioFrame] = deque()
self._audio_buffered_samples: int = 0
self._audio_sample_rate: int | None = None
self._audio_num_channels: int | None = None
# Word counts per Deepgram speaker int, accumulated across final
# transcripts inside a single user turn and cleared on turn completion.
self._pending_speakers: Counter[int] = Counter()
# Wall-clock estimate of the STT stream's t=0 (the moment Deepgram
# began consuming this participant's audio). Each FINAL gives us
# an upper bound — `now() - words[-1].end_time` is at most
# `true_t0 + final_delivery_lag` — so the MIN across FINALs
# converges to the true value. Used to translate Deepgram's
# stream-relative word offsets into absolute UTC for segment
# timestamps and to anchor the recording timeline on the server.
self._stream_t0_wall: datetime | None = None
# Min/max Deepgram-stream-relative word offsets (seconds) seen
# across FINALs within the current turn. Combined with
# `_stream_t0_wall` these give started_at/ended_at without baking
# in FINAL-delivery lag.
self._pending_first_word_offset: float | None = None
self._pending_last_word_offset: float | None = None
# Resemblyzer-backed turn-level clustering is the primary source of
# diarizedSpeakerId. Deepgram's live speaker ints are kept only as a
# fallback for turns too short/noisy to embed and as a bridge once a
# provider speaker has been associated with a voice cluster.
self._voice_cluster_resolver = VoiceClusterResolver(
channel_id=channel_id,
participant_identity=participant_identity,
)
self._provider_speaker_cluster_ids: dict[int, str] = {}
# Deepgram's speaker ints are only unique within one streaming session.
# Namespace fallback ids with a nonce regenerated per Transcriber so a
# reconnect doesn't silently merge two different humans under one UUID.
self._provider_speaker_namespace = uuid.uuid4().hex
self._provider_speaker_uuids: dict[int, str] = {}
def _resolve_provider_diarized_speaker_id(self, provider_speaker: int) -> str:
cached = self._provider_speaker_uuids.get(provider_speaker)
if cached is not None:
return cached
seed = (
f"{self.channel_id}:{self.participant_identity}:"
f"{self._provider_speaker_namespace}:provider:{provider_speaker}"
)
value = str(uuid.uuid5(uuid.NAMESPACE_URL, seed))
self._provider_speaker_uuids[provider_speaker] = value
return value
def _resolve_embedding_diarized_speaker_id(
self, embedding: list[float] | None
) -> str | None:
if not embedding:
return None
resolved = self._voice_cluster_resolver.resolve(embedding)
if resolved is None:
return None
diarized_speaker_id, nearest_distance, created_new = resolved
logger.debug(
"%s voice cluster diarizedSpeakerId=%s participant=%s "
"distance=%s threshold=%s",
"created" if created_new else "matched",
diarized_speaker_id,
self.participant_identity,
nearest_distance,
self._voice_cluster_resolver.distance_threshold,
)
return diarized_speaker_id
# Keep a bounded rolling audio buffer per participant. Resemblyzer needs
# enough speech for a useful embedding; the cap keeps memory bounded for
# long-running sessions.
_AUDIO_BUFFER_MAX_SECONDS = VOICE_EMBEDDING_BUFFER_SECONDS
def _record_audio_frame(self, frame: rtc.AudioFrame) -> None:
if self._audio_sample_rate is None:
self._audio_sample_rate = frame.sample_rate
self._audio_num_channels = frame.num_channels
self._audio_frames.append(frame)
self._audio_buffered_samples += frame.samples_per_channel
if self._audio_sample_rate:
max_samples = int(self._AUDIO_BUFFER_MAX_SECONDS * self._audio_sample_rate)
while self._audio_buffered_samples > max_samples and self._audio_frames:
dropped = self._audio_frames.popleft()
self._audio_buffered_samples -= dropped.samples_per_channel
def _drain_audio_buffer(self) -> tuple[array.array, int, int] | None:
"""Return the buffered audio as (interleaved int16 PCM, rate, channels).
Returns `None` when no audio has been captured yet. Clears the buffer
so the next turn starts fresh.
"""
if not self._audio_frames or self._audio_sample_rate is None:
return None
pcm = array.array("h")
for frame in self._audio_frames:
pcm.frombytes(bytes(frame.data))
sample_rate = self._audio_sample_rate
num_channels = self._audio_num_channels or 1
self._audio_frames.clear()
self._audio_buffered_samples = 0
return pcm, sample_rate, num_channels
# Resemblyzer needs enough speech for a usable embedding; shorter buffers
# produce noisy / unreliable vectors.
_MIN_EMBED_SECONDS = VOICE_EMBEDDING_MIN_SECONDS
async def _compute_voice_embedding(self) -> list[float] | None:
"""Drain the audio buffer and embed it with Resemblyzer.
Runs the CPU-bound preprocess+embed on a worker thread to keep the
agent event loop responsive. Returns `None` when there isn't enough
speech, when no encoder is configured, or when the model raises.
"""
if self.voice_encoder is None:
return None
drained = self._drain_audio_buffer()
if drained is None:
return None
pcm, sample_rate, num_channels = drained
if len(pcm) == 0:
return None
duration_seconds = len(pcm) / float(sample_rate * max(num_channels, 1))
if duration_seconds < self._MIN_EMBED_SECONDS:
return None
encoder = self.voice_encoder
pcm_bytes = pcm.tobytes()
def _embed() -> list[float] | None:
samples = (
np.frombuffer(pcm_bytes, dtype=np.int16).astype(np.float32) / 32768.0
)
if num_channels > 1:
samples = samples.reshape(-1, num_channels).mean(axis=1)
try:
wav = preprocess_wav(samples, source_sr=sample_rate)
except Exception:
logger.exception("preprocess_wav failed")
return None
try:
embedding = encoder.embed_utterance(wav)
except Exception:
logger.exception("voice encoder embed_utterance failed")
return None
return embedding.astype(np.float32).tolist()
return await asyncio.to_thread(_embed)
async def stt_node(self, audio, model_settings):
async def _tee():
async for frame in audio:
self._record_audio_frame(frame)
yield frame
async for event in super().stt_node(_tee(), model_settings):
if isinstance(event, stt_pkg.SpeechEvent) and (
event.type == stt_pkg.SpeechEventType.FINAL_TRANSCRIPT
):
alt = event.alternatives[0] if event.alternatives else None
words = list(getattr(alt, "words", None) or ())
if words:
first_offset = words[0].start_time
last_offset = words[-1].end_time
# Refine stream-t0 estimate. `now() - last_word.end_time`
# bounds true_t0 from above (because now() arrives at
# least final_delivery_lag after the last word ended);
# the MIN across FINALs is our best estimate.
implied_t0 = datetime.now(timezone.utc) - timedelta(
seconds=last_offset
)
if (
self._stream_t0_wall is None
or implied_t0 < self._stream_t0_wall
):
self._stream_t0_wall = implied_t0
if (
self._pending_first_word_offset is None
or first_offset < self._pending_first_word_offset
):
self._pending_first_word_offset = first_offset
if (
self._pending_last_word_offset is None
or last_offset > self._pending_last_word_offset
):
self._pending_last_word_offset = last_offset
for word in words:
speaker = getattr(word, "speaker", None)
if speaker is None:
speaker = getattr(word, "speaker_id", None)
if speaker is not None:
self._pending_speakers[int(speaker)] += 1
yield event
async def on_user_turn_completed(
self, chat_ctx: llm.ChatContext, new_message: llm.ChatMessage
):
content = new_message.text_content
# Snapshot pending state and clear unconditionally so an empty-content
# turn (e.g. VAD ended a turn with no final text) can't leak word
# offsets or speaker counts into the next turn's accumulators.
first_word_offset = self._pending_first_word_offset
last_word_offset = self._pending_last_word_offset
pending_speakers = self._pending_speakers
self._pending_first_word_offset = None
self._pending_last_word_offset = None
self._pending_speakers = Counter()
# Always drain the audio buffer at turn boundary even on empty
# content so leftover frames don't bleed into the next turn's
# embedding. _compute_voice_embedding returns None when there's
# nothing usable.
embedding = await self._compute_voice_embedding() if content else None
if not content:
self._drain_audio_buffer()
if content:
now = datetime.now(timezone.utc)
if (
self._stream_t0_wall is not None
and first_word_offset is not None
and last_word_offset is not None
):
started_at = self._stream_t0_wall + timedelta(seconds=first_word_offset)
ended_at = self._stream_t0_wall + timedelta(seconds=last_word_offset)
else:
started_at = now
ended_at = now
extra = new_message.extra if isinstance(new_message.extra, dict) else {}
source_id = (
extra.get("provider_message_id")
or extra.get("providerMessageId")
or new_message.id
)
segment_seed = (
f"{self.channel_id}:{self.participant_identity}:{source_id}:"
f"{started_at.isoformat()}"
)
segment_id = str(uuid.uuid5(uuid.NAMESPACE_URL, segment_seed))
dominant_speaker = None
if pending_speakers:
dominant_speaker, _ = pending_speakers.most_common(1)[0]
diarized_speaker_id = self._resolve_embedding_diarized_speaker_id(embedding)
if diarized_speaker_id is not None and dominant_speaker is not None:
self._provider_speaker_cluster_ids[dominant_speaker] = (
diarized_speaker_id
)
elif dominant_speaker is not None:
diarized_speaker_id = self._provider_speaker_cluster_ids.get(
dominant_speaker
) or self._resolve_provider_diarized_speaker_id(dominant_speaker)
segment = {
"segmentId": segment_id,
"speakerId": self.participant_identity,
"diarizedSpeakerId": diarized_speaker_id,
"content": content,
"startedAt": started_at.isoformat(),
"endedAt": ended_at.isoformat(),
# Sent so the server can anchor `recording_started_at` to
# the earliest first-audio-frame instant across participants
# rather than the egress webhook envelope time (which fires
# before any audio is captured).
"streamStartedAt": (
self._stream_t0_wall.isoformat()
if self._stream_t0_wall is not None
else None
),
"embedding": embedding,
"isFinal": True,
}
max_attempts = 3
delay_seconds = 0.25
for attempt in range(1, max_attempts + 1):
try:
resp = await self.http_client.post(
f"{MACRO_API_URL}/call/{self.channel_id}/transcript",
json=segment,
headers={"x-macro-internal-call": INTERNAL_CALL_SECRET},
)
resp.raise_for_status()
break
except (httpx.TimeoutException, httpx.NetworkError) as exc:
if attempt == max_attempts:
logger.exception(
"failed to post transcript segment segmentId=%s error=%s",
segment_id,
exc,
)
break
await asyncio.sleep(delay_seconds)
delay_seconds *= 2
except httpx.HTTPStatusError as exc:
status_code = exc.response.status_code if exc.response else None
is_transient = status_code is not None and 500 <= status_code < 600
if is_transient and attempt < max_attempts:
await asyncio.sleep(delay_seconds)
delay_seconds *= 2
continue
logger.exception(
"failed to post transcript segment segmentId=%s error=%s",
segment_id,
exc,
)
break
except Exception as exc:
logger.exception(
"failed to post transcript segment segmentId=%s error=%s",
segment_id,
exc,
)
break
raise StopResponse()
class MultiUserTranscriber:
"""Manages one AgentSession per participant so all speakers are transcribed."""
def __init__(self, ctx: JobContext, http_client: httpx.AsyncClient):
self.ctx = ctx
self.http_client = http_client
self._sessions: dict[str, AgentSession] = {}
self._tasks: set[asyncio.Task] = set()
def start(self):
self.ctx.room.on("participant_connected", self.on_participant_connected)
self.ctx.room.on("participant_disconnected", self.on_participant_disconnected)
async def aclose(self):
await utils.aio.cancel_and_wait(*self._tasks)
await asyncio.gather(*[self._close_session(s) for s in self._sessions.values()])
self.ctx.room.off("participant_connected", self.on_participant_connected)
self.ctx.room.off("participant_disconnected", self.on_participant_disconnected)
def on_participant_connected(self, participant: rtc.RemoteParticipant):
if participant.identity in self._sessions:
return
logger.info(f"starting transcription session for {participant.identity}")
task = asyncio.create_task(self._start_session(participant))
self._tasks.add(task)
def on_done(t: asyncio.Task):
try:
self._sessions[participant.identity] = t.result()
finally:
self._tasks.discard(t)
task.add_done_callback(on_done)
def on_participant_disconnected(self, participant: rtc.RemoteParticipant):
session = self._sessions.pop(participant.identity, None)
if session is None:
return
logger.info(f"closing transcription session for {participant.identity}")
task = asyncio.create_task(self._close_session(session))
self._tasks.add(task)
task.add_done_callback(lambda t: self._tasks.discard(t))
async def _start_session(self, participant: rtc.RemoteParticipant) -> AgentSession:
session = AgentSession(vad=self.ctx.proc.userdata["vad"])
is_first = len(self._sessions) == 0
await session.start(
agent=Transcriber(
participant_identity=participant.identity,
channel_id=self.ctx.room.name,
http_client=self.http_client,
voice_encoder=self.ctx.proc.userdata.get("voice_encoder"),
),
room=self.ctx.room,
room_options=room_io.RoomOptions(
audio_input=True,
text_input=False,
text_output=True,
audio_output=False,
participant_identity=participant.identity,
),
record=is_first,
)
return session
async def _close_session(self, session: AgentSession):
await session.drain()
await session.aclose()
async def entrypoint(ctx: JobContext):
http_client = httpx.AsyncClient()
transcriber = MultiUserTranscriber(ctx, http_client)
transcriber.start()
await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY)
for participant in ctx.room.remote_participants.values():
transcriber.on_participant_connected(participant)
async def shutdown():
await transcriber.aclose()
await http_client.aclose()
ctx.add_shutdown_callback(shutdown)
def prewarm(proc: JobProcess):
proc.userdata["vad"] = silero.VAD.load()
# Load the Resemblyzer encoder once per worker process so per-turn
# embedding doesn't pay the model-load cost on the hot path.
proc.userdata["voice_encoder"] = VoiceEncoder()
if __name__ == "__main__":
cli.run_app(
WorkerOptions(
entrypoint_fnc=entrypoint,
prewarm_fnc=prewarm,
agent_name="macro-transcriber",
# Each prewarmed process loads Silero + Resemblyzer and is expected
# to sit around ~3 GiB RSS. Keep fewer warm processes and raise the
# warning threshold so healthy workers do not spam logs every 5s.
num_idle_processes=1,
job_memory_warn_mb=4096,
)
)