Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
#!/usr/bin/env python3
"""
Tests for the Mistral/Voxtral transcription connector.
Validates:
1. Connector initialization and configuration validation
2. Capabilities and specifications
3. Segment parsing (speaker_id mapping, confidence/score)
4. Context bias (hotwords) splitting
5. Request building (diarize, language, timestamps)
6. Error handling
7. Registry integration
Run with: docker exec speakr-dev python /app/tests/test_mistral_connector.py
"""
import os
import sys
import io
import json
import re
from pathlib import Path
from unittest.mock import patch, MagicMock
sys.path.insert(0, str(Path(__file__).parent.parent))
# Test results tracking
PASSED = 0
FAILED = 0
ERRORS = []
def run_test(name, func):
global PASSED, FAILED, ERRORS
try:
func()
print(f" ✓ {name}")
PASSED += 1
except AssertionError as e:
print(f" ✗ {name}: {e}")
FAILED += 1
ERRORS.append((name, str(e)))
if "pytest" in sys.modules:
raise
except Exception as e:
print(f" ✗ {name}: EXCEPTION - {e}")
FAILED += 1
ERRORS.append((name, f"Exception: {e}"))
if "pytest" in sys.modules:
raise
# =============================================================================
# TEST SECTION 1: Initialization and Configuration
# =============================================================================
def test_initialization():
"""Test connector initialization and config validation."""
print("\n=== Testing Initialization ===")
from src.services.transcription.connectors.mistral import MistralTranscriptionConnector
from src.services.transcription.exceptions import ConfigurationError
def t1():
connector = MistralTranscriptionConnector({
'api_key': 'test-key-123',
'base_url': 'https://api.mistral.ai',
'model': 'voxtral-mini-latest',
})
assert connector.api_key == 'test-key-123'
assert connector.base_url == 'https://api.mistral.ai'
assert connector.model == 'voxtral-mini-latest'
run_test("Basic initialization with all config", t1)
def t2():
connector = MistralTranscriptionConnector({
'api_key': 'test-key',
})
assert connector.base_url == 'https://api.mistral.ai'
assert connector.model == 'voxtral-mini-latest'
run_test("Default base_url and model when not provided", t2)
def t3():
connector = MistralTranscriptionConnector({
'api_key': 'test-key',
'base_url': 'https://custom.endpoint.com/',
})
assert connector.base_url == 'https://custom.endpoint.com' # Trailing slash stripped
run_test("Trailing slash stripped from base_url", t3)
def t4():
connector = MistralTranscriptionConnector({
'api_key': 'test-key',
'base_url': '', # Empty string should fall back to default
})
assert connector.base_url == 'https://api.mistral.ai'
run_test("Empty base_url falls back to default", t4)
def t5():
try:
MistralTranscriptionConnector({'api_key': ''})
assert False, "Should have raised ConfigurationError"
except ConfigurationError as e:
assert 'api_key' in str(e)
run_test("Empty api_key raises ConfigurationError", t5)
def t6():
try:
MistralTranscriptionConnector({})
assert False, "Should have raised ConfigurationError"
except (ConfigurationError, KeyError):
pass # Either is acceptable
run_test("Missing api_key raises error", t6)
# =============================================================================
# TEST SECTION 2: Capabilities and Specifications
# =============================================================================
def test_capabilities():
"""Test connector capabilities and specifications."""
print("\n=== Testing Capabilities ===")
from src.services.transcription.connectors.mistral import MistralTranscriptionConnector
from src.services.transcription.base import TranscriptionCapability
def t1():
assert TranscriptionCapability.DIARIZATION in MistralTranscriptionConnector.CAPABILITIES
run_test("Supports diarization", t1)
def t2():
assert TranscriptionCapability.TIMESTAMPS in MistralTranscriptionConnector.CAPABILITIES
run_test("Supports timestamps", t2)
def t3():
assert TranscriptionCapability.LANGUAGE_DETECTION in MistralTranscriptionConnector.CAPABILITIES
run_test("Supports language detection", t3)
def t4():
assert TranscriptionCapability.SPEAKER_COUNT_CONTROL not in MistralTranscriptionConnector.CAPABILITIES
run_test("Does NOT support speaker count control (Mistral doesn't have min/max speakers)", t4)
def t5():
specs = MistralTranscriptionConnector.SPECIFICATIONS
assert specs.handles_chunking_internally is True
run_test("Handles chunking internally", t5)
def t6():
specs = MistralTranscriptionConnector.SPECIFICATIONS
assert specs.max_duration_seconds is None
run_test("No max_duration_seconds (avoids triggering app-level chunking)", t6)
def t7():
specs = MistralTranscriptionConnector.SPECIFICATIONS
assert specs.recommended_chunk_seconds == 0
run_test("recommended_chunk_seconds is 0 (disabled)", t7)
def t8():
specs = MistralTranscriptionConnector.SPECIFICATIONS
assert specs.max_file_size_bytes is None
run_test("No file size limit", t8)
def t9():
connector = MistralTranscriptionConnector({'api_key': 'test-key'})
assert connector.supports_diarization is True
run_test("Instance supports_diarization property", t9)
def t10():
connector = MistralTranscriptionConnector({'api_key': 'test-key'})
assert connector.PROVIDER_NAME == 'mistral'
run_test("Provider name is 'mistral'", t10)
# =============================================================================
# TEST SECTION 3: Segment Parsing
# =============================================================================
def test_segment_parsing():
"""Test _parse_segments with various Mistral response formats."""
print("\n=== Testing Segment Parsing ===")
from src.services.transcription.connectors.mistral import MistralTranscriptionConnector
connector = MistralTranscriptionConnector({'api_key': 'test-key'})
def t1():
"""Mistral uses speaker_id, not speaker."""
segments = connector._parse_segments([
{'text': 'Bonjour', 'speaker_id': 'speaker_0', 'start': 0.0, 'end': 1.5, 'score': 0.95},
{'text': 'Salut', 'speaker_id': 'speaker_1', 'start': 1.5, 'end': 3.0, 'score': 0.88},
])
assert len(segments) == 2
assert segments[0].speaker == 'speaker_0'
assert segments[1].speaker == 'speaker_1'
run_test("speaker_id field maps to speaker", t1)
def t2():
"""Fallback to 'speaker' key if speaker_id is absent."""
segments = connector._parse_segments([
{'text': 'Hello', 'speaker': 'SPEAKER_00', 'start': 0.0, 'end': 1.0},
])
assert segments[0].speaker == 'SPEAKER_00'
run_test("Falls back to 'speaker' key if speaker_id missing", t2)
def t3():
"""Score field maps to confidence."""
segments = connector._parse_segments([
{'text': 'Test', 'score': 0.92, 'start': 0.0, 'end': 1.0},
])
assert segments[0].confidence == 0.92
run_test("score field maps to confidence", t3)
def t4():
"""Timestamps are correctly parsed."""
segments = connector._parse_segments([
{'text': 'Timestamp test', 'start': 5.5, 'end': 10.2},
])
assert segments[0].start_time == 5.5
assert segments[0].end_time == 10.2
run_test("start/end timestamps are parsed", t4)
def t5():
"""Handles missing optional fields gracefully."""
segments = connector._parse_segments([
{'text': 'Minimal segment'},
])
assert len(segments) == 1
assert segments[0].text == 'Minimal segment'
assert segments[0].speaker is None
assert segments[0].start_time is None
assert segments[0].confidence is None
run_test("Missing optional fields default to None", t5)
def t6():
"""Empty segments list returns empty list."""
segments = connector._parse_segments([])
assert segments == []
run_test("Empty segments list returns empty", t6)
# =============================================================================
# TEST SECTION 4: Hotwords / Context Bias Splitting
# =============================================================================
def test_hotwords_splitting():
"""Test context_bias generation from hotwords string."""
print("\n=== Testing Hotwords Splitting ===")
def split_hotwords(hotwords_str):
"""Replicate the connector's splitting logic."""
return [w for w in re.split(r'[,\s]+', hotwords_str) if w]
def t1():
result = split_hotwords("Speakr, Voxtral, PyAnnote")
assert result == ['Speakr', 'Voxtral', 'PyAnnote']
run_test("Comma-separated hotwords split correctly", t1)
def t2():
result = split_hotwords("Speakr Voxtral PyAnnote")
assert result == ['Speakr', 'Voxtral', 'PyAnnote']
run_test("Space-separated hotwords split correctly", t2)
def t3():
result = split_hotwords("Speakr,Voxtral,PyAnnote")
assert result == ['Speakr', 'Voxtral', 'PyAnnote']
run_test("Comma-only (no spaces) hotwords split correctly", t3)
def t4():
result = split_hotwords(" Speakr , Voxtral , PyAnnote ")
assert result == ['Speakr', 'Voxtral', 'PyAnnote']
run_test("Extra whitespace is trimmed", t4)
def t5():
result = split_hotwords("")
assert result == []
run_test("Empty string produces empty list", t5)
def t6():
result = split_hotwords("SingleWord")
assert result == ['SingleWord']
run_test("Single word produces one-item list", t6)
def t7():
r"""Each item must match ^[^,\s]+$ per Mistral spec."""
result = split_hotwords("hello world, foo bar, baz")
for item in result:
assert re.match(r'^[^,\s]+$', item), f"'{item}' contains comma or whitespace"
run_test("All split items conform to Mistral's single-token pattern", t7)
# =============================================================================
# TEST SECTION 5: Request Building
# =============================================================================
def test_request_building():
"""Test that transcribe() builds the correct multipart request."""
print("\n=== Testing Request Building ===")
from src.services.transcription.connectors.mistral import MistralTranscriptionConnector
from src.services.transcription.base import TranscriptionRequest
connector = MistralTranscriptionConnector({'api_key': 'test-key'})
def make_request(**kwargs):
defaults = {
'audio_file': io.BytesIO(b'fake audio'),
'filename': 'test.wav',
}
defaults.update(kwargs)
return TranscriptionRequest(**defaults)
def t1():
"""Verify diarize and timestamp_granularities are always sent together."""
captured = {}
def mock_post(url, **kwargs):
captured['url'] = url
captured['files'] = kwargs.get('files', [])
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.json.return_value = {'text': 'test', 'segments': []}
return mock_resp
connector.client.post = mock_post
req = make_request(diarize=True)
connector.transcribe(req)
field_names = [f[0] for f in captured['files']]
assert 'diarize' in field_names, f"diarize not in fields: {field_names}"
assert 'timestamp_granularities' in field_names, f"timestamp_granularities not in fields: {field_names}"
run_test("Diarize sends both diarize and timestamp_granularities", t1)
def t2():
"""Language is passed through when provided."""
captured = {}
def mock_post(url, **kwargs):
captured['files'] = kwargs.get('files', [])
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.json.return_value = {'text': 'test', 'segments': []}
return mock_resp
connector.client.post = mock_post
req = make_request(language='fr')
connector.transcribe(req)
fields = {f[0]: f[1] for f in captured['files'] if f[0] != 'file'}
assert 'language' in fields
# Value is a tuple (None, 'fr') for multipart encoding
assert fields['language'] == (None, 'fr')
run_test("Language parameter is sent when provided", t2)
def t3():
"""Hotwords are sent as individual context_bias fields."""
captured = {}
def mock_post(url, **kwargs):
captured['files'] = kwargs.get('files', [])
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.json.return_value = {'text': 'test', 'segments': []}
return mock_resp
connector.client.post = mock_post
req = make_request(hotwords='Speakr, Voxtral, Python')
connector.transcribe(req)
context_bias_fields = [(name, val) for name, val in captured['files'] if name == 'context_bias']
assert len(context_bias_fields) == 3, f"Expected 3 context_bias fields, got {len(context_bias_fields)}"
run_test("Hotwords are sent as individual context_bias fields", t3)
def t4():
"""Model is always sent."""
captured = {}
def mock_post(url, **kwargs):
captured['files'] = kwargs.get('files', [])
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.json.return_value = {'text': 'test', 'segments': []}
return mock_resp
connector.client.post = mock_post
req = make_request()
connector.transcribe(req)
field_names = [f[0] for f in captured['files']]
assert 'model' in field_names
run_test("Model is always included in request", t4)
def t5():
"""Prompt parameter is ignored (Mistral doesn't support it)."""
captured = {}
def mock_post(url, **kwargs):
captured['files'] = kwargs.get('files', [])
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.json.return_value = {'text': 'test', 'segments': []}
return mock_resp
connector.client.post = mock_post
req = make_request(prompt='This is a prompt')
connector.transcribe(req)
field_names = [f[0] for f in captured['files']]
assert 'prompt' not in field_names
assert 'initial_prompt' not in field_names
run_test("Prompt parameter is NOT sent to Mistral", t5)
# =============================================================================
# TEST SECTION 6: Response Parsing
# =============================================================================
def test_response_parsing():
"""Test full response parsing from a mock Mistral API response."""
print("\n=== Testing Response Parsing ===")
from src.services.transcription.connectors.mistral import MistralTranscriptionConnector
from src.services.transcription.base import TranscriptionRequest
connector = MistralTranscriptionConnector({'api_key': 'test-key'})
def t1():
"""Full diarized response with speaker_id and score."""
mock_response = {
'text': 'Bonjour comment allez-vous? Très bien merci.',
'language': 'fr',
'segments': [
{'text': 'Bonjour comment allez-vous?', 'speaker_id': 'speaker_0', 'start': 0.0, 'end': 2.5, 'score': 0.95},
{'text': 'Très bien merci.', 'speaker_id': 'speaker_1', 'start': 2.5, 'end': 4.0, 'score': 0.88},
],
'usage': {'prompt_tokens': 100, 'total_tokens': 100},
}
def mock_post(url, **kwargs):
resp = MagicMock()
resp.status_code = 200
resp.json.return_value = mock_response
return resp
connector.client.post = mock_post
req = TranscriptionRequest(audio_file=io.BytesIO(b'fake'), filename='test.wav', diarize=True)
result = connector.transcribe(req)
assert result.text == 'Bonjour comment allez-vous? Très bien merci.'
assert result.language == 'fr'
assert result.provider == 'mistral'
assert len(result.segments) == 2
assert result.segments[0].speaker == 'speaker_0'
assert result.segments[1].speaker == 'speaker_1'
assert result.segments[0].confidence == 0.95
assert result.speakers == ['speaker_0', 'speaker_1']
run_test("Full diarized response parsed correctly", t1)
def t2():
"""Response without segments (plain text only)."""
mock_response = {
'text': 'Hello world',
'language': 'en',
}
def mock_post(url, **kwargs):
resp = MagicMock()
resp.status_code = 200
resp.json.return_value = mock_response
return resp
connector.client.post = mock_post
req = TranscriptionRequest(audio_file=io.BytesIO(b'fake'), filename='test.wav')
result = connector.transcribe(req)
assert result.text == 'Hello world'
assert result.segments == []
assert result.speakers is None
run_test("Plain text response (no segments) parsed correctly", t2)
def t3():
"""has_diarization() returns True only when segments have speakers."""
mock_response = {
'text': 'Test',
'segments': [
{'text': 'Test', 'speaker_id': 'speaker_0', 'start': 0.0, 'end': 1.0},
],
}
def mock_post(url, **kwargs):
resp = MagicMock()
resp.status_code = 200
resp.json.return_value = mock_response
return resp
connector.client.post = mock_post
req = TranscriptionRequest(audio_file=io.BytesIO(b'fake'), filename='test.wav', diarize=True)
result = connector.transcribe(req)
assert result.has_diarization() is True
run_test("has_diarization() returns True for diarized response", t3)
# =============================================================================
# TEST SECTION 7: Error Handling
# =============================================================================
def test_error_handling():
"""Test error handling for API failures."""
print("\n=== Testing Error Handling ===")
from src.services.transcription.connectors.mistral import MistralTranscriptionConnector
from src.services.transcription.base import TranscriptionRequest
from src.services.transcription.exceptions import ProviderError, TranscriptionError
connector = MistralTranscriptionConnector({'api_key': 'test-key'})
def t1():
"""Non-200 response raises ProviderError."""
def mock_post(url, **kwargs):
resp = MagicMock()
resp.status_code = 422
resp.text = '{"detail": "Invalid request"}'
resp.json.return_value = {'detail': 'Invalid request'}
return resp
connector.client.post = mock_post
req = TranscriptionRequest(audio_file=io.BytesIO(b'fake'), filename='test.wav')
try:
connector.transcribe(req)
assert False, "Should have raised ProviderError"
except ProviderError as e:
assert e.provider == 'mistral'
assert e.status_code == 422
run_test("API 422 error raises ProviderError with details", t1)
def t2():
"""Timeout raises TranscriptionError."""
import httpx
def mock_post(url, **kwargs):
raise httpx.TimeoutException("Connection timed out")
connector.client.post = mock_post
req = TranscriptionRequest(audio_file=io.BytesIO(b'fake'), filename='test.wav')
try:
connector.transcribe(req)
assert False, "Should have raised TranscriptionError"
except TranscriptionError as e:
assert 'timed out' in str(e)
run_test("Timeout raises TranscriptionError", t2)
def t3():
"""Health check returns True with valid api_key."""
connector2 = MistralTranscriptionConnector({'api_key': 'valid-key'})
assert connector2.health_check() is True
run_test("health_check() returns True with valid key", t3)
def t4():
"""Health check returns False with empty api_key (if bypassing validation)."""
# Directly set empty key to test health_check logic
connector2 = MistralTranscriptionConnector.__new__(MistralTranscriptionConnector)
connector2.config = {'api_key': ''}
assert connector2.health_check() is False
run_test("health_check() returns False with empty key", t4)
# =============================================================================
# TEST SECTION 8: Registry Integration
# =============================================================================
def test_registry_integration():
"""Test that Mistral connector is properly registered."""
print("\n=== Testing Registry Integration ===")
def clear_env():
keys = [
'TRANSCRIPTION_CONNECTOR', 'TRANSCRIPTION_API_KEY', 'TRANSCRIPTION_BASE_URL',
'TRANSCRIPTION_MODEL', 'USE_ASR_ENDPOINT', 'ASR_BASE_URL',
]
for key in keys:
os.environ.pop(key, None)
def reset_registry():
from src.services.transcription import registry
registry._registry = None
registry.ConnectorRegistry._instance = None
registry.ConnectorRegistry._initialized = False
registry.ConnectorRegistry._active_connector = None
registry.ConnectorRegistry._connector_name = ""
from src.services.transcription.registry import get_registry
def t1():
clear_env()
reset_registry()
registry = get_registry()
connectors = registry.list_connectors()
names = [c['name'] for c in connectors]
assert 'mistral' in names, f"'mistral' not in registered connectors: {names}"
run_test("Mistral connector is registered in registry", t1)
def t2():
clear_env()
reset_registry()
registry = get_registry()
connectors = registry.list_connectors()
mistral = next(c for c in connectors if c['name'] == 'mistral')
assert 'DIARIZATION' in mistral['capabilities']
assert 'TIMESTAMPS' in mistral['capabilities']
assert 'LANGUAGE_DETECTION' in mistral['capabilities']
run_test("Mistral connector info shows correct capabilities", t2)
def t3():
clear_env()
reset_registry()
os.environ['TRANSCRIPTION_CONNECTOR'] = 'mistral'
os.environ['TRANSCRIPTION_API_KEY'] = 'test-key'
registry = get_registry()
registry.initialize_from_env()
assert registry.get_active_connector_name() == 'mistral'
run_test("TRANSCRIPTION_CONNECTOR=mistral activates Mistral connector", t3)
def t4():
clear_env()
reset_registry()
os.environ['TRANSCRIPTION_CONNECTOR'] = 'mistral'
os.environ['TRANSCRIPTION_API_KEY'] = 'test-key'
os.environ['TRANSCRIPTION_MODEL'] = 'voxtral-mini-2602'
registry = get_registry()
connector = registry.initialize_from_env()
assert connector.model == 'voxtral-mini-2602'
run_test("Custom model name is passed to connector", t4)
def t5():
clear_env()
reset_registry()
os.environ['TRANSCRIPTION_CONNECTOR'] = 'mistral'
os.environ['TRANSCRIPTION_API_KEY'] = 'test-key'
os.environ['TRANSCRIPTION_BASE_URL'] = 'https://custom.endpoint.com # comment'
registry = get_registry()
connector = registry.initialize_from_env()
assert connector.base_url == 'https://custom.endpoint.com'
run_test("Trailing comment stripped from TRANSCRIPTION_BASE_URL", t5)
# Cleanup
clear_env()
# =============================================================================
# TEST SECTION 9: Chunking Behavior
# =============================================================================
def test_chunking_behavior():
"""Test that Mistral connector specs correctly prevent app-level chunking."""
print("\n=== Testing Chunking Behavior ===")
from src.audio_chunking import get_effective_chunking_config
from src.services.transcription.connectors.mistral import MistralTranscriptionConnector
def t1():
os.environ['ENABLE_CHUNKING'] = 'true'
os.environ['CHUNK_LIMIT'] = '20MB'
config = get_effective_chunking_config(MistralTranscriptionConnector.SPECIFICATIONS)
assert config.enabled is False, f"Chunking should be disabled for Mistral but got enabled={config.enabled}"
assert config.source == 'connector_internal'
os.environ.pop('ENABLE_CHUNKING', None)
os.environ.pop('CHUNK_LIMIT', None)
run_test("Mistral specs disable app-level chunking", t1)
def t2():
# MISTRAL_ENABLE_CHUNKING=true should opt the connector into app-side chunking.
os.environ['MISTRAL_ENABLE_CHUNKING'] = 'true'
os.environ['MISTRAL_MAX_DURATION_SECONDS'] = '3600'
try:
connector = MistralTranscriptionConnector({'api_key': 'test'})
specs = connector.SPECIFICATIONS
assert specs.handles_chunking_internally is False, \
f"Chunking should be opt-in but got handles_chunking_internally={specs.handles_chunking_internally}"
assert specs.max_duration_seconds == 3600, f"got max_duration_seconds={specs.max_duration_seconds}"
# 80% of max duration as the recommended chunk size.
assert specs.recommended_chunk_seconds == 2880, \
f"got recommended_chunk_seconds={specs.recommended_chunk_seconds}"
finally:
os.environ.pop('MISTRAL_ENABLE_CHUNKING', None)
os.environ.pop('MISTRAL_MAX_DURATION_SECONDS', None)
run_test("MISTRAL_ENABLE_CHUNKING=true opts in to app-side chunking", t2)
# =============================================================================
# Main
# =============================================================================
def main():
global PASSED, FAILED, ERRORS
print("=" * 60)
print("Mistral/Voxtral Transcription Connector Tests")
print("=" * 60)
test_initialization()
test_capabilities()
test_segment_parsing()
test_hotwords_splitting()
test_request_building()
test_response_parsing()
test_error_handling()
test_registry_integration()
test_chunking_behavior()
print("\n" + "=" * 60)
print(f"RESULTS: {PASSED} passed, {FAILED} failed")
print("=" * 60)
if ERRORS:
print("\nFailed tests:")
for name, error in ERRORS:
print(f" - {name}: {error}")
return 0 if FAILED == 0 else 1
if __name__ == '__main__':
sys.exit(main())