Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

Add AssemblyAI transcription connector (async upload/poll API, diarization, speaker hints, word boost) with config docs and unit tests

Murtaza Nasir68b47392f373
diff --git a/config/env.transcription.example b/config/env.transcription.example
index 737b3941..26821a03 100644
--- a/config/env.transcription.example
+++ b/config/env.transcription.example
@@ -62,7 +62,7 @@ GPT5_VERBOSITY=medium
# =============================================================================
# CONNECTOR SELECTION (Auto-detected if not set)
# =============================================================================
-# Options: openai_whisper, openai_transcribe, asr_endpoint, mistral, vibevoice
+# Options: openai_whisper, openai_transcribe, asr_endpoint, mistral, vibevoice, assemblyai
# Leave empty to auto-detect based on other settings
# TRANSCRIPTION_CONNECTOR=
@@ -162,6 +162,28 @@ TRANSCRIPTION_MODEL=gpt-4o-transcribe-diarize
# If your vLLM server requires authentication:
# TRANSCRIPTION_API_KEY=your_api_key
+# =============================================================================
+# ASSEMBLYAI CONFIGURATION (Cloud)
+# =============================================================================
+# AssemblyAI is a cloud transcription provider with strong speaker diarization
+# on long, multi-speaker recordings. It is NOT OpenAI-compatible (it uses its
+# own async upload/poll API), so it has its own connector. It ingests files up
+# to 10 hours in a single job, so the app does not chunk for it.
+#
+# New accounts get free credits and no credit card is required to get a key
+# (sign up at https://www.assemblyai.com and create an API key).
+#
+# TRANSCRIPTION_CONNECTOR=assemblyai
+# TRANSCRIPTION_API_KEY=your_assemblyai_api_key
+#
+# Optional: pin a speech model (omit to use your account default).
+# TRANSCRIPTION_MODEL=universal
+#
+# Diarization, language, and hotwords come from each user's settings (or tag /
+# folder defaults). Hotwords are sent as AssemblyAI word_boost; min/max speaker
+# counts map to speaker_options. Times are returned in milliseconds and stored
+# as seconds.
+
# =============================================================================
# CHUNKING CONFIGURATION (For large files)
# =============================================================================
diff --git a/src/services/transcription/connectors/assemblyai.py b/src/services/transcription/connectors/assemblyai.py
new file mode 100644
index 00000000..d36cdae9
--- /dev/null
+++ b/src/services/transcription/connectors/assemblyai.py
@@ -0,0 +1,268 @@
+"""
+AssemblyAI transcription connector.
+
+AssemblyAI is not OpenAI-compatible: it uses an async upload-then-poll REST API
+(`/v2/upload` -> `/v2/transcript` -> poll `/v2/transcript/{id}`) with its own
+diarization output (an `utterances` array, times in milliseconds). It handles
+long multi-speaker files in a single job (up to 10 hours), so no app-side
+chunking is needed.
+
+Config (from env via the registry):
+- api_key: TRANSCRIPTION_API_KEY (required; sent as the `authorization` header, no Bearer)
+- base_url: TRANSCRIPTION_BASE_URL (default https://api.assemblyai.com)
+- model: TRANSCRIPTION_MODEL (optional; passed as `speech_model`, omitted when blank)
+"""
+
+import io
+import time
+import logging
+from typing import Dict, Any, Set, Optional, List
+
+import httpx
+
+from ..base import (
+ BaseTranscriptionConnector,
+ TranscriptionCapability,
+ TranscriptionRequest,
+ TranscriptionResponse,
+ TranscriptionSegment,
+ ConnectorSpecifications,
+)
+from ..exceptions import TranscriptionError, ConfigurationError, ProviderError
+
+logger = logging.getLogger(__name__)
+
+
+class AssemblyAITranscriptionConnector(BaseTranscriptionConnector):
+ """Connector for AssemblyAI's async transcription API with diarization."""
+
+ CAPABILITIES: Set[TranscriptionCapability] = {
+ TranscriptionCapability.DIARIZATION,
+ TranscriptionCapability.TIMESTAMPS,
+ TranscriptionCapability.LANGUAGE_DETECTION,
+ TranscriptionCapability.SPEAKER_COUNT_CONTROL, # speaker_options min/max
+ TranscriptionCapability.HOTWORDS, # word_boost
+ }
+ PROVIDER_NAME = "assemblyai"
+
+ # AssemblyAI ingests long files in a single async job (10h / up to 2.2GB on
+ # upload), so the app must not chunk for this connector.
+ SPECIFICATIONS = ConnectorSpecifications(
+ max_file_size_bytes=None,
+ max_duration_seconds=None,
+ handles_chunking_internally=True,
+ )
+
+ def __init__(self, config: Dict[str, Any]):
+ self.api_key = (config.get('api_key') or '').strip()
+ base = (config.get('base_url') or 'https://api.assemblyai.com').strip()
+ self.base_url = base.rstrip('/')
+ self.model = (config.get('model') or '').strip()
+ # Poll behaviour: AssemblyAI jobs for long meetings take minutes, so the
+ # ceiling is generous. Both are overridable via config for tuning/tests.
+ self.poll_interval = float(config.get('poll_interval', 3.0))
+ self.poll_timeout = float(config.get('poll_timeout', 3600.0)) # 1 hour
+ # `boost_param` strength for word_boost (low | default | high).
+ self.boost_param = (config.get('boost_param') or 'default').strip()
+ super().__init__(config)
+
+ def _validate_config(self) -> None:
+ if not self.api_key:
+ raise ConfigurationError("api_key is required for the AssemblyAI connector (set TRANSCRIPTION_API_KEY)")
+
+ # -- HTTP helpers -------------------------------------------------------
+
+ def _headers(self) -> Dict[str, str]:
+ # AssemblyAI takes the raw key in `authorization`, with NO Bearer prefix.
+ return {"authorization": self.api_key}
+
+ def _client(self) -> httpx.Client:
+ return httpx.Client(base_url=self.base_url, headers=self._headers(), timeout=60.0)
+
+ def _upload(self, client: httpx.Client, audio_bytes: bytes) -> str:
+ resp = client.post("/v2/upload", content=audio_bytes,
+ headers={"content-type": "application/octet-stream"})
+ if resp.status_code != 200:
+ raise ProviderError(f"AssemblyAI upload failed ({resp.status_code}): {resp.text[:300]}",
+ provider=self.PROVIDER_NAME, status_code=resp.status_code)
+ url = resp.json().get('upload_url')
+ if not url:
+ raise ProviderError("AssemblyAI upload returned no upload_url", provider=self.PROVIDER_NAME)
+ return url
+
+ def _build_payload(self, request: TranscriptionRequest, audio_url: str) -> Dict[str, Any]:
+ payload: Dict[str, Any] = {
+ "audio_url": audio_url,
+ "punctuate": True,
+ "format_text": True,
+ }
+
+ # Language: an explicit code disables auto-detection; otherwise detect.
+ lang = (request.language or '').strip().lower()
+ if lang and lang not in ('auto', 'any', 'detect'):
+ payload["language_code"] = request.language
+ else:
+ payload["language_detection"] = True
+
+ # Diarization + speaker-count hinting.
+ if request.diarize:
+ payload["speaker_labels"] = True
+ mn, mx = request.min_speakers, request.max_speakers
+ if mn and mx and int(mn) == int(mx):
+ payload["speakers_expected"] = int(mn)
+ else:
+ opts = {}
+ if mn:
+ opts["min_speakers_expected"] = int(mn)
+ if mx:
+ opts["max_speakers_expected"] = int(mx)
+ if opts:
+ payload["speaker_options"] = opts
+
+ # Hotwords -> word_boost (comma-separated string to list).
+ if request.hotwords:
+ words = [w.strip() for w in request.hotwords.split(',') if w.strip()]
+ if words:
+ payload["word_boost"] = words
+ payload["boost_param"] = self.boost_param
+
+ # Optional model override (account default when blank).
+ model = self._effective_model(request)
+ if model:
+ payload["speech_model"] = model
+
+ return payload
+
+ def _poll(self, client: httpx.Client, transcript_id: str) -> Dict[str, Any]:
+ deadline = time.monotonic() + self.poll_timeout
+ while True:
+ resp = client.get(f"/v2/transcript/{transcript_id}")
+ if resp.status_code != 200:
+ raise ProviderError(
+ f"AssemblyAI poll failed ({resp.status_code}): {resp.text[:300]}",
+ provider=self.PROVIDER_NAME, status_code=resp.status_code)
+ data = resp.json()
+ status = data.get('status')
+ if status == 'completed':
+ return data
+ if status == 'error':
+ raise TranscriptionError(f"AssemblyAI transcription failed: {data.get('error', 'unknown error')}")
+ if time.monotonic() > deadline:
+ raise TranscriptionError(
+ f"AssemblyAI transcription timed out after {self.poll_timeout:.0f}s "
+ f"(transcript {transcript_id} still '{status}')")
+ time.sleep(self.poll_interval)
+
+ # -- Main entry ---------------------------------------------------------
+
+ def transcribe(self, request: TranscriptionRequest) -> TranscriptionResponse:
+ try:
+ audio_bytes = self._read_audio(request)
+ with self._client() as client:
+ audio_url = self._upload(client, audio_bytes)
+ payload = self._build_payload(request, audio_url)
+ logger.info(
+ "AssemblyAI: submitting transcript (diarize=%s, lang=%s, model=%s)",
+ request.diarize, payload.get('language_code', 'auto'), payload.get('speech_model', 'default'))
+ resp = client.post("/v2/transcript", json=payload,
+ headers={"content-type": "application/json"})
+ if resp.status_code not in (200, 201):
+ raise ProviderError(
+ f"AssemblyAI submit failed ({resp.status_code}): {resp.text[:300]}",
+ provider=self.PROVIDER_NAME, status_code=resp.status_code)
+ transcript_id = resp.json().get('id')
+ if not transcript_id:
+ raise ProviderError("AssemblyAI submit returned no transcript id", provider=self.PROVIDER_NAME)
+ result = self._poll(client, transcript_id)
+ return self._parse_result(result)
+ except (TranscriptionError, ConfigurationError):
+ raise
+ except Exception as e:
+ logger.error("AssemblyAI transcription failed: %s", e)
+ raise TranscriptionError(f"AssemblyAI transcription failed: {e}") from e
+
+ @staticmethod
+ def _read_audio(request: TranscriptionRequest) -> bytes:
+ f = request.audio_file
+ try:
+ if hasattr(f, 'seek'):
+ f.seek(0)
+ except (OSError, io.UnsupportedOperation):
+ pass
+ data = f.read()
+ if not data:
+ raise TranscriptionError("AssemblyAI: empty audio file")
+ return data
+
+ def _parse_result(self, result: Dict[str, Any]) -> TranscriptionResponse:
+ text = result.get('text') or ''
+ language = result.get('language_code')
+ duration = result.get('audio_duration') # seconds
+
+ utterances = result.get('utterances')
+ if utterances:
+ segments: List[TranscriptionSegment] = []
+ speakers = set()
+ text_parts = []
+ for u in utterances:
+ spk = u.get('speaker') or 'Unknown'
+ utext = (u.get('text') or '').strip()
+ if not utext:
+ continue
+ speakers.add(spk)
+ text_parts.append(f"[{spk}]: {utext}")
+ segments.append(TranscriptionSegment(
+ text=utext,
+ speaker=spk,
+ start_time=_ms_to_s(u.get('start')),
+ end_time=_ms_to_s(u.get('end')),
+ confidence=u.get('confidence'),
+ ))
+ logger.info("AssemblyAI: %d utterances, %d speakers", len(segments), len(speakers))
+ return TranscriptionResponse(
+ text='\n'.join(text_parts) if text_parts else text,
+ segments=segments,
+ speakers=sorted(speakers),
+ language=language,
+ duration=duration,
+ provider=self.PROVIDER_NAME,
+ model=self.model or 'default',
+ raw_response=result,
+ )
+
+ # No diarization: plain text.
+ return TranscriptionResponse(
+ text=text,
+ language=language,
+ duration=duration,
+ provider=self.PROVIDER_NAME,
+ model=self.model or 'default',
+ raw_response=result,
+ )
+
+ def health_check(self) -> bool:
+ return bool(self.api_key)
+
+ @classmethod
+ def get_config_schema(cls) -> Dict[str, Any]:
+ return {
+ "type": "object",
+ "required": ["api_key"],
+ "properties": {
+ "api_key": {"type": "string", "description": "AssemblyAI API key"},
+ "base_url": {"type": "string", "default": "https://api.assemblyai.com",
+ "description": "API base URL"},
+ "model": {"type": "string", "default": "",
+ "description": "Optional speech_model (account default when blank)"},
+ },
+ }
+
+
+def _ms_to_s(ms: Optional[int]) -> Optional[float]:
+ """AssemblyAI reports times in milliseconds; the app stores seconds."""
+ if ms is None:
+ return None
+ try:
+ return round(ms / 1000.0, 3)
+ except (TypeError, ValueError):
+ return None
diff --git a/src/services/transcription/registry.py b/src/services/transcription/registry.py
index 899a31bc..292ace07 100644
--- a/src/services/transcription/registry.py
+++ b/src/services/transcription/registry.py
@@ -47,6 +47,7 @@ class ConnectorRegistry:
from .connectors.azure_openai_transcribe import AzureOpenAITranscribeConnector
from .connectors.mistral import MistralTranscriptionConnector
from .connectors.vibevoice import VibeVoiceTranscriptionConnector
+ from .connectors.assemblyai import AssemblyAITranscriptionConnector
self.register('openai_whisper', OpenAIWhisperConnector)
self.register('openai_transcribe', OpenAITranscribeConnector)
@@ -54,6 +55,7 @@ class ConnectorRegistry:
self.register('azure_openai_transcribe', AzureOpenAITranscribeConnector)
self.register('mistral', MistralTranscriptionConnector)
self.register('vibevoice', VibeVoiceTranscriptionConnector)
+ self.register('assemblyai', AssemblyAITranscriptionConnector)
def register(self, name: str, connector_class: Type[BaseTranscriptionConnector]):
"""
@@ -289,6 +291,18 @@ class ConnectorRegistry:
'api_key': os.environ.get('TRANSCRIPTION_API_KEY', ''),
}
+ elif connector_name == 'assemblyai':
+ base_url = os.environ.get('TRANSCRIPTION_BASE_URL', '')
+ if base_url:
+ base_url = base_url.split('#')[0].strip()
+
+ return {
+ 'api_key': os.environ.get('TRANSCRIPTION_API_KEY', ''),
+ 'base_url': base_url or 'https://api.assemblyai.com',
+ # Optional speech_model; account default when blank.
+ 'model': os.environ.get('TRANSCRIPTION_MODEL', ''),
+ }
+
else: # openai_whisper (default)
base_url = os.environ.get('TRANSCRIPTION_BASE_URL', '')
if base_url:
diff --git a/tests/test_cov_assemblyai_connector.py b/tests/test_cov_assemblyai_connector.py
new file mode 100644
index 00000000..8a4ea794
--- /dev/null
+++ b/tests/test_cov_assemblyai_connector.py
@@ -0,0 +1,185 @@
+"""
+Coverage for the AssemblyAI transcription connector.
+
+AssemblyAI uses an async upload -> submit -> poll REST flow with diarization
+output in an `utterances` array (times in milliseconds). These tests mock
+httpx so no network is touched, and verify: registration, capabilities, config
+validation, request-payload shaping (diarization, speaker hints, hotwords,
+language), the poll loop (processing -> completed and -> error), and the
+milliseconds-to-seconds utterance parsing.
+"""
+
+import io
+import json
+import pytest
+from unittest.mock import patch
+
+import src.services.transcription.connectors.assemblyai as aai_mod
+from src.services.transcription.connectors.assemblyai import AssemblyAITranscriptionConnector
+from src.services.transcription.base import (
+ TranscriptionRequest, TranscriptionCapability,
+)
+from src.services.transcription.exceptions import TranscriptionError, ConfigurationError
+
+
+# ---------------------------------------------------------------------------
+# Fake httpx client: scripts canned responses for upload/submit/poll by path.
+# ---------------------------------------------------------------------------
+
+class _Resp:
+ def __init__(self, status_code=200, payload=None, text=""):
+ self.status_code = status_code
+ self._payload = payload or {}
+ self.text = text
+ def json(self):
+ return self._payload
+
+
+class _FakeClient:
+ def __init__(self, poll_statuses=None, final=None, submit_status=200):
+ # poll_statuses: list of status strings returned by successive GETs;
+ # the final 'completed' GET returns `final`.
+ self.poll_statuses = list(poll_statuses or ['completed'])
+ self.final = final or {}
+ self.submit_status = submit_status
+ self.posted = [] # (path, json/content)
+ def __enter__(self): return self
+ def __exit__(self, *a): return False
+ def post(self, path, content=None, json=None, headers=None):
+ self.posted.append((path, json if json is not None else content))
+ if path == '/v2/upload':
+ return _Resp(200, {'upload_url': 'https://cdn.assemblyai.com/upload/x'})
+ if path == '/v2/transcript':
+ return _Resp(self.submit_status, {'id': 'tid-1', 'status': 'queued'})
+ raise AssertionError(f"unexpected POST {path}")
+ def get(self, path):
+ status = self.poll_statuses.pop(0)
+ if status == 'completed':
+ return _Resp(200, dict(self.final, status='completed'))
+ if status == 'error':
+ return _Resp(200, {'status': 'error', 'error': 'boom'})
+ return _Resp(200, {'status': status})
+
+
+def _conn(**cfg):
+ base = {'api_key': 'fake', 'poll_interval': 0}
+ base.update(cfg)
+ return AssemblyAITranscriptionConnector(base)
+
+
+def _req(**kw):
+ kw.setdefault('audio_file', io.BytesIO(b'audio-bytes'))
+ kw.setdefault('filename', 'a.mp3')
+ return TranscriptionRequest(**kw)
+
+
+# --- registration / capabilities / config ---------------------------------
+
+def test_registered_in_registry():
+ from src.services.transcription.registry import get_registry
+ names = [c['name'] for c in get_registry().list_connectors()]
+ assert 'assemblyai' in names
+
+
+def test_capabilities():
+ caps = _conn().get_capabilities()
+ assert TranscriptionCapability.DIARIZATION in caps
+ assert TranscriptionCapability.HOTWORDS in caps
+ assert TranscriptionCapability.SPEAKER_COUNT_CONTROL in caps
+ assert TranscriptionCapability.LANGUAGE_DETECTION in caps
+ # No free-text initial prompt on AssemblyAI base transcription.
+ assert TranscriptionCapability.INITIAL_PROMPT not in caps
+
+
+def test_handles_chunking_internally():
+ assert _conn().specifications.handles_chunking_internally is True
+
+
+def test_missing_api_key_raises():
+ with pytest.raises(ConfigurationError):
+ AssemblyAITranscriptionConnector({'api_key': ''})
+
+
+# --- payload shaping -------------------------------------------------------
+
+def test_payload_diarize_range_hotwords_language():
+ p = _conn()._build_payload(
+ _req(language='en', diarize=True, min_speakers=2, max_speakers=5,
+ hotwords='Speakr, PyAnnote'),
+ 'u')
+ assert p['language_code'] == 'en'
+ assert p['speaker_labels'] is True
+ assert p['speaker_options'] == {'min_speakers_expected': 2, 'max_speakers_expected': 5}
+ assert p['word_boost'] == ['Speakr', 'PyAnnote'] and p['boost_param'] == 'default'
+ assert 'language_detection' not in p
+
+
+def test_payload_equal_min_max_uses_speakers_expected():
+ p = _conn()._build_payload(_req(language='auto', diarize=True, min_speakers=3, max_speakers=3), 'u')
+ assert p['speakers_expected'] == 3
+ assert 'speaker_options' not in p
+ assert p['language_detection'] is True # 'auto' -> detection
+
+
+def test_payload_no_diarize_no_speaker_fields():
+ p = _conn()._build_payload(_req(language='de', diarize=False), 'u')
+ assert 'speaker_labels' not in p and 'speaker_options' not in p
+ assert p['language_code'] == 'de'
+
+
+def test_payload_model_passthrough():
+ p = _conn(model='universal')._build_payload(_req(diarize=False, language='en'), 'u')
+ assert p['speech_model'] == 'universal'
+
+
+# --- end-to-end flow (mocked) ---------------------------------------------
+
+DIARIZED_FINAL = {
+ 'text': 'Hello there Hi back',
+ 'language_code': 'en',
+ 'audio_duration': 12.5,
+ 'utterances': [
+ {'speaker': 'A', 'text': 'Hello there', 'start': 250, 'end': 2950, 'confidence': 0.93},
+ {'speaker': 'B', 'text': 'Hi back', 'start': 3000, 'end': 4100, 'confidence': 0.88},
+ ],
+}
+
+
+def test_transcribe_diarized_flow_parses_utterances():
+ fake = _FakeClient(poll_statuses=['processing', 'completed'], final=DIARIZED_FINAL)
+ c = _conn()
+ with patch.object(c, '_client', return_value=fake):
+ resp = c.transcribe(_req(language='en', diarize=True))
+ assert [(s.speaker, s.text, s.start_time, s.end_time) for s in resp.segments] == [
+ ('A', 'Hello there', 0.25, 2.95),
+ ('B', 'Hi back', 3.0, 4.1),
+ ]
+ assert resp.speakers == ['A', 'B']
+ assert resp.duration == 12.5
+ assert resp.provider == 'assemblyai'
+ # storage format round-trips with ms converted to seconds
+ stored = json.loads(resp.to_storage_format())
+ assert stored[0] == {'speaker': 'A', 'sentence': 'Hello there', 'start_time': 0.25, 'end_time': 2.95}
+ # uploaded the audio bytes and submitted with speaker_labels
+ paths = [p for p, _ in fake.posted]
+ assert paths == ['/v2/upload', '/v2/transcript']
+ assert fake.posted[1][1]['speaker_labels'] is True
+
+
+def test_transcribe_plain_text_when_no_utterances():
+ fake = _FakeClient(poll_statuses=['completed'],
+ final={'text': 'just text', 'language_code': 'en', 'audio_duration': 3.0})
+ c = _conn()
+ with patch.object(c, '_client', return_value=fake):
+ resp = c.transcribe(_req(language='en', diarize=False))
+ assert resp.segments is None
+ assert resp.text == 'just text'
+ assert resp.to_storage_format() == 'just text'
+
+
+def test_transcribe_error_status_raises():
+ fake = _FakeClient(poll_statuses=['processing', 'error'])
+ c = _conn()
+ with patch.object(c, '_client', return_value=fake):
+ with pytest.raises(TranscriptionError, match='boom'):
+ c.transcribe(_req(diarize=True))