Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
Add AssemblyAI transcription connector (async upload/poll API, diarization, speaker hints, word boost) with config docs and unit tests
Murtaza Nasir68b47392f373
diff --git a/config/env.transcription.example b/config/env.transcription.exampleindex 737b3941..26821a03 100644--- a/config/env.transcription.example+++ b/config/env.transcription.example@@ -62,7 +62,7 @@ GPT5_VERBOSITY=medium# =============================================================================# CONNECTOR SELECTION (Auto-detected if not set)# =============================================================================-# Options: openai_whisper, openai_transcribe, asr_endpoint, mistral, vibevoice+# Options: openai_whisper, openai_transcribe, asr_endpoint, mistral, vibevoice, assemblyai# Leave empty to auto-detect based on other settings# TRANSCRIPTION_CONNECTOR=@@ -162,6 +162,28 @@ TRANSCRIPTION_MODEL=gpt-4o-transcribe-diarize# If your vLLM server requires authentication:# TRANSCRIPTION_API_KEY=your_api_key+# =============================================================================+# ASSEMBLYAI CONFIGURATION (Cloud)+# =============================================================================+# AssemblyAI is a cloud transcription provider with strong speaker diarization+# on long, multi-speaker recordings. It is NOT OpenAI-compatible (it uses its+# own async upload/poll API), so it has its own connector. It ingests files up+# to 10 hours in a single job, so the app does not chunk for it.+#+# New accounts get free credits and no credit card is required to get a key+# (sign up at https://www.assemblyai.com and create an API key).+#+# TRANSCRIPTION_CONNECTOR=assemblyai+# TRANSCRIPTION_API_KEY=your_assemblyai_api_key+#+# Optional: pin a speech model (omit to use your account default).+# TRANSCRIPTION_MODEL=universal+#+# Diarization, language, and hotwords come from each user's settings (or tag /+# folder defaults). Hotwords are sent as AssemblyAI word_boost; min/max speaker+# counts map to speaker_options. Times are returned in milliseconds and stored+# as seconds.+# =============================================================================# CHUNKING CONFIGURATION (For large files)# =============================================================================diff --git a/src/services/transcription/connectors/assemblyai.py b/src/services/transcription/connectors/assemblyai.pynew file mode 100644index 00000000..d36cdae9--- /dev/null+++ b/src/services/transcription/connectors/assemblyai.py@@ -0,0 +1,268 @@+"""+AssemblyAI transcription connector.++AssemblyAI is not OpenAI-compatible: it uses an async upload-then-poll REST API+(`/v2/upload` -> `/v2/transcript` -> poll `/v2/transcript/{id}`) with its own+diarization output (an `utterances` array, times in milliseconds). It handles+long multi-speaker files in a single job (up to 10 hours), so no app-side+chunking is needed.++Config (from env via the registry):+- api_key: TRANSCRIPTION_API_KEY (required; sent as the `authorization` header, no Bearer)+- base_url: TRANSCRIPTION_BASE_URL (default https://api.assemblyai.com)+- model: TRANSCRIPTION_MODEL (optional; passed as `speech_model`, omitted when blank)+"""++import io+import time+import logging+from typing import Dict, Any, Set, Optional, List++import httpx++from ..base import (+ BaseTranscriptionConnector,+ TranscriptionCapability,+ TranscriptionRequest,+ TranscriptionResponse,+ TranscriptionSegment,+ ConnectorSpecifications,+)+from ..exceptions import TranscriptionError, ConfigurationError, ProviderError++logger = logging.getLogger(__name__)+++class AssemblyAITranscriptionConnector(BaseTranscriptionConnector):+ """Connector for AssemblyAI's async transcription API with diarization."""++ CAPABILITIES: Set[TranscriptionCapability] = {+ TranscriptionCapability.DIARIZATION,+ TranscriptionCapability.TIMESTAMPS,+ TranscriptionCapability.LANGUAGE_DETECTION,+ TranscriptionCapability.SPEAKER_COUNT_CONTROL, # speaker_options min/max+ TranscriptionCapability.HOTWORDS, # word_boost+ }+ PROVIDER_NAME = "assemblyai"++ # AssemblyAI ingests long files in a single async job (10h / up to 2.2GB on+ # upload), so the app must not chunk for this connector.+ SPECIFICATIONS = ConnectorSpecifications(+ max_file_size_bytes=None,+ max_duration_seconds=None,+ handles_chunking_internally=True,+ )++ def __init__(self, config: Dict[str, Any]):+ self.api_key = (config.get('api_key') or '').strip()+ base = (config.get('base_url') or 'https://api.assemblyai.com').strip()+ self.base_url = base.rstrip('/')+ self.model = (config.get('model') or '').strip()+ # Poll behaviour: AssemblyAI jobs for long meetings take minutes, so the+ # ceiling is generous. Both are overridable via config for tuning/tests.+ self.poll_interval = float(config.get('poll_interval', 3.0))+ self.poll_timeout = float(config.get('poll_timeout', 3600.0)) # 1 hour+ # `boost_param` strength for word_boost (low | default | high).+ self.boost_param = (config.get('boost_param') or 'default').strip()+ super().__init__(config)++ def _validate_config(self) -> None:+ if not self.api_key:+ raise ConfigurationError("api_key is required for the AssemblyAI connector (set TRANSCRIPTION_API_KEY)")++ # -- HTTP helpers -------------------------------------------------------++ def _headers(self) -> Dict[str, str]:+ # AssemblyAI takes the raw key in `authorization`, with NO Bearer prefix.+ return {"authorization": self.api_key}++ def _client(self) -> httpx.Client:+ return httpx.Client(base_url=self.base_url, headers=self._headers(), timeout=60.0)++ def _upload(self, client: httpx.Client, audio_bytes: bytes) -> str:+ resp = client.post("/v2/upload", content=audio_bytes,+ headers={"content-type": "application/octet-stream"})+ if resp.status_code != 200:+ raise ProviderError(f"AssemblyAI upload failed ({resp.status_code}): {resp.text[:300]}",+ provider=self.PROVIDER_NAME, status_code=resp.status_code)+ url = resp.json().get('upload_url')+ if not url:+ raise ProviderError("AssemblyAI upload returned no upload_url", provider=self.PROVIDER_NAME)+ return url++ def _build_payload(self, request: TranscriptionRequest, audio_url: str) -> Dict[str, Any]:+ payload: Dict[str, Any] = {+ "audio_url": audio_url,+ "punctuate": True,+ "format_text": True,+ }++ # Language: an explicit code disables auto-detection; otherwise detect.+ lang = (request.language or '').strip().lower()+ if lang and lang not in ('auto', 'any', 'detect'):+ payload["language_code"] = request.language+ else:+ payload["language_detection"] = True++ # Diarization + speaker-count hinting.+ if request.diarize:+ payload["speaker_labels"] = True+ mn, mx = request.min_speakers, request.max_speakers+ if mn and mx and int(mn) == int(mx):+ payload["speakers_expected"] = int(mn)+ else:+ opts = {}+ if mn:+ opts["min_speakers_expected"] = int(mn)+ if mx:+ opts["max_speakers_expected"] = int(mx)+ if opts:+ payload["speaker_options"] = opts++ # Hotwords -> word_boost (comma-separated string to list).+ if request.hotwords:+ words = [w.strip() for w in request.hotwords.split(',') if w.strip()]+ if words:+ payload["word_boost"] = words+ payload["boost_param"] = self.boost_param++ # Optional model override (account default when blank).+ model = self._effective_model(request)+ if model:+ payload["speech_model"] = model++ return payload++ def _poll(self, client: httpx.Client, transcript_id: str) -> Dict[str, Any]:+ deadline = time.monotonic() + self.poll_timeout+ while True:+ resp = client.get(f"/v2/transcript/{transcript_id}")+ if resp.status_code != 200:+ raise ProviderError(+ f"AssemblyAI poll failed ({resp.status_code}): {resp.text[:300]}",+ provider=self.PROVIDER_NAME, status_code=resp.status_code)+ data = resp.json()+ status = data.get('status')+ if status == 'completed':+ return data+ if status == 'error':+ raise TranscriptionError(f"AssemblyAI transcription failed: {data.get('error', 'unknown error')}")+ if time.monotonic() > deadline:+ raise TranscriptionError(+ f"AssemblyAI transcription timed out after {self.poll_timeout:.0f}s "+ f"(transcript {transcript_id} still '{status}')")+ time.sleep(self.poll_interval)++ # -- Main entry ---------------------------------------------------------++ def transcribe(self, request: TranscriptionRequest) -> TranscriptionResponse:+ try:+ audio_bytes = self._read_audio(request)+ with self._client() as client:+ audio_url = self._upload(client, audio_bytes)+ payload = self._build_payload(request, audio_url)+ logger.info(+ "AssemblyAI: submitting transcript (diarize=%s, lang=%s, model=%s)",+ request.diarize, payload.get('language_code', 'auto'), payload.get('speech_model', 'default'))+ resp = client.post("/v2/transcript", json=payload,+ headers={"content-type": "application/json"})+ if resp.status_code not in (200, 201):+ raise ProviderError(+ f"AssemblyAI submit failed ({resp.status_code}): {resp.text[:300]}",+ provider=self.PROVIDER_NAME, status_code=resp.status_code)+ transcript_id = resp.json().get('id')+ if not transcript_id:+ raise ProviderError("AssemblyAI submit returned no transcript id", provider=self.PROVIDER_NAME)+ result = self._poll(client, transcript_id)+ return self._parse_result(result)+ except (TranscriptionError, ConfigurationError):+ raise+ except Exception as e:+ logger.error("AssemblyAI transcription failed: %s", e)+ raise TranscriptionError(f"AssemblyAI transcription failed: {e}") from e++ @staticmethod+ def _read_audio(request: TranscriptionRequest) -> bytes:+ f = request.audio_file+ try:+ if hasattr(f, 'seek'):+ f.seek(0)+ except (OSError, io.UnsupportedOperation):+ pass+ data = f.read()+ if not data:+ raise TranscriptionError("AssemblyAI: empty audio file")+ return data++ def _parse_result(self, result: Dict[str, Any]) -> TranscriptionResponse:+ text = result.get('text') or ''+ language = result.get('language_code')+ duration = result.get('audio_duration') # seconds++ utterances = result.get('utterances')+ if utterances:+ segments: List[TranscriptionSegment] = []+ speakers = set()+ text_parts = []+ for u in utterances:+ spk = u.get('speaker') or 'Unknown'+ utext = (u.get('text') or '').strip()+ if not utext:+ continue+ speakers.add(spk)+ text_parts.append(f"[{spk}]: {utext}")+ segments.append(TranscriptionSegment(+ text=utext,+ speaker=spk,+ start_time=_ms_to_s(u.get('start')),+ end_time=_ms_to_s(u.get('end')),+ confidence=u.get('confidence'),+ ))+ logger.info("AssemblyAI: %d utterances, %d speakers", len(segments), len(speakers))+ return TranscriptionResponse(+ text='\n'.join(text_parts) if text_parts else text,+ segments=segments,+ speakers=sorted(speakers),+ language=language,+ duration=duration,+ provider=self.PROVIDER_NAME,+ model=self.model or 'default',+ raw_response=result,+ )++ # No diarization: plain text.+ return TranscriptionResponse(+ text=text,+ language=language,+ duration=duration,+ provider=self.PROVIDER_NAME,+ model=self.model or 'default',+ raw_response=result,+ )++ def health_check(self) -> bool:+ return bool(self.api_key)++ @classmethod+ def get_config_schema(cls) -> Dict[str, Any]:+ return {+ "type": "object",+ "required": ["api_key"],+ "properties": {+ "api_key": {"type": "string", "description": "AssemblyAI API key"},+ "base_url": {"type": "string", "default": "https://api.assemblyai.com",+ "description": "API base URL"},+ "model": {"type": "string", "default": "",+ "description": "Optional speech_model (account default when blank)"},+ },+ }+++def _ms_to_s(ms: Optional[int]) -> Optional[float]:+ """AssemblyAI reports times in milliseconds; the app stores seconds."""+ if ms is None:+ return None+ try:+ return round(ms / 1000.0, 3)+ except (TypeError, ValueError):+ return Nonediff --git a/src/services/transcription/registry.py b/src/services/transcription/registry.pyindex 899a31bc..292ace07 100644--- a/src/services/transcription/registry.py+++ b/src/services/transcription/registry.py@@ -47,6 +47,7 @@ class ConnectorRegistry:from .connectors.azure_openai_transcribe import AzureOpenAITranscribeConnectorfrom .connectors.mistral import MistralTranscriptionConnectorfrom .connectors.vibevoice import VibeVoiceTranscriptionConnector+ from .connectors.assemblyai import AssemblyAITranscriptionConnectorself.register('openai_whisper', OpenAIWhisperConnector)self.register('openai_transcribe', OpenAITranscribeConnector)@@ -54,6 +55,7 @@ class ConnectorRegistry:self.register('azure_openai_transcribe', AzureOpenAITranscribeConnector)self.register('mistral', MistralTranscriptionConnector)self.register('vibevoice', VibeVoiceTranscriptionConnector)+ self.register('assemblyai', AssemblyAITranscriptionConnector)def register(self, name: str, connector_class: Type[BaseTranscriptionConnector]):"""@@ -289,6 +291,18 @@ class ConnectorRegistry:'api_key': os.environ.get('TRANSCRIPTION_API_KEY', ''),}+ elif connector_name == 'assemblyai':+ base_url = os.environ.get('TRANSCRIPTION_BASE_URL', '')+ if base_url:+ base_url = base_url.split('#')[0].strip()++ return {+ 'api_key': os.environ.get('TRANSCRIPTION_API_KEY', ''),+ 'base_url': base_url or 'https://api.assemblyai.com',+ # Optional speech_model; account default when blank.+ 'model': os.environ.get('TRANSCRIPTION_MODEL', ''),+ }+else: # openai_whisper (default)base_url = os.environ.get('TRANSCRIPTION_BASE_URL', '')if base_url:diff --git a/tests/test_cov_assemblyai_connector.py b/tests/test_cov_assemblyai_connector.pynew file mode 100644index 00000000..8a4ea794--- /dev/null+++ b/tests/test_cov_assemblyai_connector.py@@ -0,0 +1,185 @@+"""+Coverage for the AssemblyAI transcription connector.++AssemblyAI uses an async upload -> submit -> poll REST flow with diarization+output in an `utterances` array (times in milliseconds). These tests mock+httpx so no network is touched, and verify: registration, capabilities, config+validation, request-payload shaping (diarization, speaker hints, hotwords,+language), the poll loop (processing -> completed and -> error), and the+milliseconds-to-seconds utterance parsing.+"""++import io+import json+import pytest+from unittest.mock import patch++import src.services.transcription.connectors.assemblyai as aai_mod+from src.services.transcription.connectors.assemblyai import AssemblyAITranscriptionConnector+from src.services.transcription.base import (+ TranscriptionRequest, TranscriptionCapability,+)+from src.services.transcription.exceptions import TranscriptionError, ConfigurationError+++# ---------------------------------------------------------------------------+# Fake httpx client: scripts canned responses for upload/submit/poll by path.+# ---------------------------------------------------------------------------++class _Resp:+ def __init__(self, status_code=200, payload=None, text=""):+ self.status_code = status_code+ self._payload = payload or {}+ self.text = text+ def json(self):+ return self._payload+++class _FakeClient:+ def __init__(self, poll_statuses=None, final=None, submit_status=200):+ # poll_statuses: list of status strings returned by successive GETs;+ # the final 'completed' GET returns `final`.+ self.poll_statuses = list(poll_statuses or ['completed'])+ self.final = final or {}+ self.submit_status = submit_status+ self.posted = [] # (path, json/content)+ def __enter__(self): return self+ def __exit__(self, *a): return False+ def post(self, path, content=None, json=None, headers=None):+ self.posted.append((path, json if json is not None else content))+ if path == '/v2/upload':+ return _Resp(200, {'upload_url': 'https://cdn.assemblyai.com/upload/x'})+ if path == '/v2/transcript':+ return _Resp(self.submit_status, {'id': 'tid-1', 'status': 'queued'})+ raise AssertionError(f"unexpected POST {path}")+ def get(self, path):+ status = self.poll_statuses.pop(0)+ if status == 'completed':+ return _Resp(200, dict(self.final, status='completed'))+ if status == 'error':+ return _Resp(200, {'status': 'error', 'error': 'boom'})+ return _Resp(200, {'status': status})+++def _conn(**cfg):+ base = {'api_key': 'fake', 'poll_interval': 0}+ base.update(cfg)+ return AssemblyAITranscriptionConnector(base)+++def _req(**kw):+ kw.setdefault('audio_file', io.BytesIO(b'audio-bytes'))+ kw.setdefault('filename', 'a.mp3')+ return TranscriptionRequest(**kw)+++# --- registration / capabilities / config ---------------------------------++def test_registered_in_registry():+ from src.services.transcription.registry import get_registry+ names = [c['name'] for c in get_registry().list_connectors()]+ assert 'assemblyai' in names+++def test_capabilities():+ caps = _conn().get_capabilities()+ assert TranscriptionCapability.DIARIZATION in caps+ assert TranscriptionCapability.HOTWORDS in caps+ assert TranscriptionCapability.SPEAKER_COUNT_CONTROL in caps+ assert TranscriptionCapability.LANGUAGE_DETECTION in caps+ # No free-text initial prompt on AssemblyAI base transcription.+ assert TranscriptionCapability.INITIAL_PROMPT not in caps+++def test_handles_chunking_internally():+ assert _conn().specifications.handles_chunking_internally is True+++def test_missing_api_key_raises():+ with pytest.raises(ConfigurationError):+ AssemblyAITranscriptionConnector({'api_key': ''})+++# --- payload shaping -------------------------------------------------------++def test_payload_diarize_range_hotwords_language():+ p = _conn()._build_payload(+ _req(language='en', diarize=True, min_speakers=2, max_speakers=5,+ hotwords='Speakr, PyAnnote'),+ 'u')+ assert p['language_code'] == 'en'+ assert p['speaker_labels'] is True+ assert p['speaker_options'] == {'min_speakers_expected': 2, 'max_speakers_expected': 5}+ assert p['word_boost'] == ['Speakr', 'PyAnnote'] and p['boost_param'] == 'default'+ assert 'language_detection' not in p+++def test_payload_equal_min_max_uses_speakers_expected():+ p = _conn()._build_payload(_req(language='auto', diarize=True, min_speakers=3, max_speakers=3), 'u')+ assert p['speakers_expected'] == 3+ assert 'speaker_options' not in p+ assert p['language_detection'] is True # 'auto' -> detection+++def test_payload_no_diarize_no_speaker_fields():+ p = _conn()._build_payload(_req(language='de', diarize=False), 'u')+ assert 'speaker_labels' not in p and 'speaker_options' not in p+ assert p['language_code'] == 'de'+++def test_payload_model_passthrough():+ p = _conn(model='universal')._build_payload(_req(diarize=False, language='en'), 'u')+ assert p['speech_model'] == 'universal'+++# --- end-to-end flow (mocked) ---------------------------------------------++DIARIZED_FINAL = {+ 'text': 'Hello there Hi back',+ 'language_code': 'en',+ 'audio_duration': 12.5,+ 'utterances': [+ {'speaker': 'A', 'text': 'Hello there', 'start': 250, 'end': 2950, 'confidence': 0.93},+ {'speaker': 'B', 'text': 'Hi back', 'start': 3000, 'end': 4100, 'confidence': 0.88},+ ],+}+++def test_transcribe_diarized_flow_parses_utterances():+ fake = _FakeClient(poll_statuses=['processing', 'completed'], final=DIARIZED_FINAL)+ c = _conn()+ with patch.object(c, '_client', return_value=fake):+ resp = c.transcribe(_req(language='en', diarize=True))+ assert [(s.speaker, s.text, s.start_time, s.end_time) for s in resp.segments] == [+ ('A', 'Hello there', 0.25, 2.95),+ ('B', 'Hi back', 3.0, 4.1),+ ]+ assert resp.speakers == ['A', 'B']+ assert resp.duration == 12.5+ assert resp.provider == 'assemblyai'+ # storage format round-trips with ms converted to seconds+ stored = json.loads(resp.to_storage_format())+ assert stored[0] == {'speaker': 'A', 'sentence': 'Hello there', 'start_time': 0.25, 'end_time': 2.95}+ # uploaded the audio bytes and submitted with speaker_labels+ paths = [p for p, _ in fake.posted]+ assert paths == ['/v2/upload', '/v2/transcript']+ assert fake.posted[1][1]['speaker_labels'] is True+++def test_transcribe_plain_text_when_no_utterances():+ fake = _FakeClient(poll_statuses=['completed'],+ final={'text': 'just text', 'language_code': 'en', 'audio_duration': 3.0})+ c = _conn()+ with patch.object(c, '_client', return_value=fake):+ resp = c.transcribe(_req(language='en', diarize=False))+ assert resp.segments is None+ assert resp.text == 'just text'+ assert resp.to_storage_format() == 'just text'+++def test_transcribe_error_status_raises():+ fake = _FakeClient(poll_statuses=['processing', 'error'])+ c = _conn()+ with patch.object(c, '_client', return_value=fake):+ with pytest.raises(TranscriptionError, match='boom'):+ c.transcribe(_req(diarize=True))