Skip to content

StreamVault is a TV-first free IPTV player for Android TV built with Kotlin, Jetpack Compose, Room, Hilt, and Media3.

Add realtime translation feature with live audio tap and settings integration

a4289438569a7
diff --git a/app/src/main/java/com/streamvault/app/player/LiveTranslationClient.kt b/app/src/main/java/com/streamvault/app/player/LiveTranslationClient.kt
new file mode 100644
index 00000000..4fe5ba7b
--- /dev/null
+++ b/app/src/main/java/com/streamvault/app/player/LiveTranslationClient.kt
@@ -0,0 +1,104 @@
+package com.streamvault.app.player
+
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import okhttp3.RequestBody.Companion.toRequestBody
+import org.json.JSONObject
+
+private val JSON_MEDIA_TYPE = "application/json; charset=utf-8".toMediaType()
+private val PCM_MEDIA_TYPE = "application/octet-stream".toMediaType()
+private const val DEFAULT_WHISPER_MODEL = "whisper-large-v3"
+
+/**
+ * A streaming translation update for the current phrase. [chunkId] identifies the
+ * phrase: partials (isFinal=false) update the same line in place, a new chunkId
+ * starts a new line, and isFinal marks the phrase as complete. [text] may be blank
+ * when an upload produced no new transcription (silence / in-progress).
+ */
+data class LiveTranslationUpdate(
+ val chunkId: Long,
+ val isFinal: Boolean,
+ val text: String,
+ val sourceLanguage: String?
+)
+
+class LiveTranslationClient(
+ private val okHttpClient: OkHttpClient,
+ private val endpoint: String
+) {
+ suspend fun startSession(
+ logicalUrl: String,
+ providerId: Long,
+ contentId: Long
+ ): String = withContext(Dispatchers.IO) {
+ val requestPayload = JSONObject()
+ .put("logicalStreamUrl", logicalUrl)
+ .put("providerId", providerId)
+ .put("contentId", contentId)
+ .put("targetLanguage", "en")
+ .put("task", "translate")
+ .put("model", DEFAULT_WHISPER_MODEL)
+ .put("autoDetectSourceLanguage", true)
+ val request = Request.Builder()
+ .url("${endpoint.trimEnd('/')}/v1/live-translation/session")
+ .post(requestPayload.toString().toRequestBody(JSON_MEDIA_TYPE))
+ .build()
+ okHttpClient.newCall(request).execute().use { response ->
+ if (!response.isSuccessful) {
+ throw IllegalStateException("Live translation start failed (${response.code})")
+ }
+ val body = response.body?.string().orEmpty()
+ val json = JSONObject(body.ifBlank { "{}" })
+ json.optString("sessionId")
+ .ifBlank { json.optString("id") }
+ .ifBlank { throw IllegalStateException("Missing translation session id") }
+ }
+ }
+
+ suspend fun uploadPcmChunk(
+ sessionId: String,
+ pcm16Mono16k: ByteArray,
+ startMs: Long,
+ endMs: Long
+ ): LiveTranslationUpdate =
+ withContext(Dispatchers.IO) {
+ val url = "${endpoint.trimEnd('/')}/v1/live-translation/session/$sessionId/audio" +
+ "?startMs=$startMs&endMs=$endMs&sampleRate=16000&channelCount=1&encoding=pcm_s16le"
+ val request = Request.Builder()
+ .url(url)
+ .post(pcm16Mono16k.toRequestBody(PCM_MEDIA_TYPE))
+ .build()
+ okHttpClient.newCall(request).execute().use { response ->
+ if (!response.isSuccessful) {
+ throw IllegalStateException("Live translation audio upload failed (${response.code})")
+ }
+ val body = response.body?.string().orEmpty()
+ parseTranslationUpdate(body)
+ }
+ }
+
+ suspend fun stopSession(sessionId: String) {
+ withContext(Dispatchers.IO) {
+ val request = Request.Builder()
+ .url("${endpoint.trimEnd('/')}/v1/live-translation/session/$sessionId")
+ .delete()
+ .build()
+ runCatching {
+ okHttpClient.newCall(request).execute().use { /* best-effort */ }
+ }
+ }
+ }
+}
+
+internal fun parseTranslationUpdate(body: String): LiveTranslationUpdate {
+ val json = JSONObject(body.ifBlank { "{}" })
+ return LiveTranslationUpdate(
+ chunkId = json.optLong("chunkId"),
+ isFinal = json.optBoolean("isFinal"),
+ text = json.optString("text").trim(),
+ sourceLanguage = json.optString("sourceLanguage").takeIf { it.isNotBlank() }
+ )
+}
diff --git a/app/src/main/java/com/streamvault/app/player/LiveTranslationSession.kt b/app/src/main/java/com/streamvault/app/player/LiveTranslationSession.kt
new file mode 100644
index 00000000..10618755
--- /dev/null
+++ b/app/src/main/java/com/streamvault/app/player/LiveTranslationSession.kt
@@ -0,0 +1,250 @@
+package com.streamvault.app.player
+
+import androidx.media3.common.C
+import androidx.media3.common.text.Cue
+import com.streamvault.player.LiveAudioPcmBuffer
+import com.streamvault.player.PlayerEngine
+import java.io.ByteArrayOutputStream
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.channels.BufferOverflow
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.isActive
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.withTimeoutOrNull
+
+private const val TARGET_SAMPLE_RATE = 16_000
+private const val TARGET_CHANNEL_COUNT = 1
+private const val TARGET_BYTES_PER_SAMPLE = 2
+// Upload small audio increments frequently; the service accumulates them into a
+// phrase, emits in-progress partials, and finalises on a natural pause (VAD).
+private const val UPLOAD_CHUNK_MS = 1_000L
+private const val MAX_PENDING_AUDIO_BUFFERS = 120
+// Minimum time a caption stays on screen before it can advance to newer text, so
+// the subtitle doesn't change faster than a viewer can read. Trades a little
+// latency for readability; intermediate updates during this window are coalesced.
+private const val MIN_CAPTION_DISPLAY_MS = 2_200L
+// Clear the caption after this much idle time with no new text (TV-caption feel).
+private const val SUBTITLE_LINGER_MS = 20_000L
+
+private data class CaptionTick(
+ val text: String,
+ val isFinal: Boolean,
+ val chunkId: Long
+)
+
+class LiveTranslationSession(
+ private val scope: CoroutineScope,
+ private val playerEngine: PlayerEngine,
+ private val client: LiveTranslationClient,
+ private val logicalUrl: String,
+ private val providerId: Long,
+ private val contentId: Long,
+ private val onSourceLanguageDetected: (String) -> Unit,
+ private val onError: (String) -> Unit
+) {
+ private var sessionId: String? = null
+ private var pollingJob: Job? = null
+ private var audioBuffers = newAudioChannel()
+ // Latest pending caption text waiting to be displayed. Conflated: if several
+ // updates arrive during a dwell, only the most recent is kept.
+ private var captionUpdates = newCaptionChannel()
+ private var hasVisibleCaption = false
+
+ fun start() {
+ stop()
+ audioBuffers = newAudioChannel()
+ captionUpdates = newCaptionChannel()
+ playerEngine.setLiveAudioTap { buffer ->
+ audioBuffers.trySend(buffer)
+ }
+ pollingJob = scope.launch {
+ runCatching {
+ sessionId = client.startSession(
+ logicalUrl = logicalUrl,
+ providerId = providerId,
+ contentId = contentId
+ )
+ coroutineScope {
+ launch { displayLoop() }
+ audioUploadLoop()
+ }
+ }.onFailure {
+ if (it is CancellationException) {
+ clearCues()
+ return@onFailure
+ }
+ onError("Live translation unavailable: ${it.message.orEmpty().ifBlank { "service error" }}")
+ clearCues()
+ }
+ }
+ }
+
+ fun stop() {
+ pollingJob?.cancel()
+ pollingJob = null
+ playerEngine.clearLiveAudioTap()
+ audioBuffers.close()
+ captionUpdates.close()
+ val currentSessionId = sessionId
+ sessionId = null
+ clearCues()
+ if (currentSessionId != null) {
+ scope.launch {
+ client.stopSession(currentSessionId)
+ }
+ }
+ }
+
+ private suspend fun audioUploadLoop() {
+ val chunk = ByteArrayOutputStream()
+ var chunkStartMs: Long? = null
+ var chunkEndMs = 0L
+ var fallbackPositionMs = playerEngine.currentPosition.value
+ for (buffer in audioBuffers) {
+ if (!scope.isActive) return
+ val activeSessionId = sessionId ?: return
+ val converted = convertToPcm16Mono16k(buffer, fallbackPositionMs) ?: continue
+ fallbackPositionMs = converted.endMs
+ if (converted.data.isEmpty()) continue
+ val activeChunkStartMs = chunkStartMs ?: converted.startMs
+ chunkStartMs = activeChunkStartMs
+ chunkEndMs = converted.endMs
+ chunk.write(converted.data)
+ if (chunkEndMs - activeChunkStartMs < UPLOAD_CHUNK_MS) {
+ continue
+ }
+ uploadChunk(activeSessionId, chunk.toByteArray(), activeChunkStartMs, chunkEndMs)
+ chunk.reset()
+ chunkStartMs = null
+ delay(50L)
+ }
+ }
+
+ private suspend fun uploadChunk(sessionId: String, pcmChunk: ByteArray, startMs: Long, endMs: Long) {
+ val update = client.uploadPcmChunk(
+ sessionId = sessionId,
+ pcm16Mono16k = pcmChunk,
+ startMs = startMs,
+ endMs = endMs
+ )
+ // Empty text means silence / no new transcription for this upload.
+ if (update.text.isBlank()) {
+ return
+ }
+ update.sourceLanguage?.let(onSourceLanguageDetected)
+ // Hand the latest text to the display loop, which paces how fast captions
+ // change. Conflated channel: if we're mid-dwell, this just replaces the
+ // pending text so the viewer always advances to the most recent state.
+ captionUpdates.trySend(
+ CaptionTick(
+ text = update.text,
+ isFinal = update.isFinal,
+ chunkId = update.chunkId
+ )
+ )
+ }
+
+ /**
+ * Paces caption changes for readability. Partial lines update immediately and
+ * any queued updates are drained in order so finals aren't skipped. Finalised
+ * phrases dwell for [MIN_CAPTION_DISPLAY_MS]. Clears after [SUBTITLE_LINGER_MS]
+ * of no updates.
+ */
+ private suspend fun displayLoop() {
+ while (scope.isActive) {
+ val result = withTimeoutOrNull(SUBTITLE_LINGER_MS) { captionUpdates.receiveCatching() }
+ if (result == null) {
+ if (hasVisibleCaption) clearCues()
+ continue
+ }
+ val tick = result.getOrNull() ?: break // channel closed -> session ended
+ showCaptionTick(tick)
+ drainQueuedCaptionTicks()
+ }
+ }
+
+ private suspend fun showCaptionTick(tick: CaptionTick) {
+ if (tick.text.isBlank()) return
+ renderCaption(tick.text)
+ hasVisibleCaption = true
+ if (tick.isFinal) {
+ delay(MIN_CAPTION_DISPLAY_MS)
+ }
+ }
+
+ /** Show any captions that arrived while the previous tick was on screen. */
+ private suspend fun drainQueuedCaptionTicks() {
+ while (scope.isActive) {
+ val next = captionUpdates.tryReceive().getOrNull() ?: break
+ showCaptionTick(next)
+ }
+ }
+
+ private fun renderCaption(text: String) {
+ playerEngine.setInjectedSubtitleCues(
+ listOf(Cue.Builder().setText(text).build())
+ )
+ }
+
+ private fun clearCues() {
+ hasVisibleCaption = false
+ playerEngine.clearInjectedSubtitleCues()
+ }
+}
+
+private data class ConvertedPcmChunk(
+ val data: ByteArray,
+ val startMs: Long,
+ val endMs: Long
+)
+
+private fun newAudioChannel(): Channel<LiveAudioPcmBuffer> = Channel(
+ capacity = MAX_PENDING_AUDIO_BUFFERS,
+ onBufferOverflow = BufferOverflow.DROP_OLDEST
+)
+
+private fun newCaptionChannel(): Channel<CaptionTick> = Channel(
+ capacity = 64,
+ onBufferOverflow = BufferOverflow.DROP_OLDEST
+)
+
+private fun convertToPcm16Mono16k(buffer: LiveAudioPcmBuffer, fallbackStartMs: Long): ConvertedPcmChunk? {
+ if (buffer.encoding != C.ENCODING_PCM_16BIT || buffer.sampleRate <= 0 || buffer.channelCount <= 0) {
+ return null
+ }
+ val inputFrameSize = buffer.channelCount * TARGET_BYTES_PER_SAMPLE
+ val inputFrames = buffer.data.size / inputFrameSize
+ if (inputFrames <= 0) return null
+
+ val outputFrames = ((inputFrames.toLong() * TARGET_SAMPLE_RATE) / buffer.sampleRate)
+ .coerceAtLeast(1L)
+ .toInt()
+ val output = ByteArray(outputFrames * TARGET_CHANNEL_COUNT * TARGET_BYTES_PER_SAMPLE)
+ for (outputFrame in 0 until outputFrames) {
+ val inputFrame = ((outputFrame.toLong() * buffer.sampleRate) / TARGET_SAMPLE_RATE)
+ .coerceIn(0L, (inputFrames - 1).toLong())
+ .toInt()
+ var mixed = 0
+ for (channel in 0 until buffer.channelCount) {
+ val byteIndex = inputFrame * inputFrameSize + channel * TARGET_BYTES_PER_SAMPLE
+ val low = buffer.data[byteIndex].toInt() and 0xFF
+ val high = buffer.data[byteIndex + 1].toInt()
+ mixed += (high shl 8) or low
+ }
+ val sample = (mixed / buffer.channelCount).coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt())
+ val outputIndex = outputFrame * TARGET_BYTES_PER_SAMPLE
+ output[outputIndex] = (sample and 0xFF).toByte()
+ output[outputIndex + 1] = ((sample shr 8) and 0xFF).toByte()
+ }
+ val startMs = fallbackStartMs.coerceAtLeast(0L)
+ val durationMs = (outputFrames * 1_000L) / TARGET_SAMPLE_RATE
+ return ConvertedPcmChunk(
+ data = output,
+ startMs = startMs,
+ endMs = startMs + durationMs
+ )
+}
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/epg/EpgViewModel.kt b/app/src/main/java/com/streamvault/app/ui/screens/epg/EpgViewModel.kt
index 27b9b110..cf2e4c1d 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/epg/EpgViewModel.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/epg/EpgViewModel.kt
@@ -537,6 +537,14 @@ class EpgViewModel @Inject constructor(
}
fun scheduleRecording(channel: Channel, program: Program, recurrence: RecordingRecurrence = RecordingRecurrence.NONE) {
+ // When the program is airing now, capture starts immediately. Free the provider connection
+ // the guide preview is holding, or on single-connection Xtream accounts the capture would
+ // be a second connection and get a 403. Recording from the guide means "record without
+ // watching", so dropping the preview is intended; it only restarts on an explicit click.
+ // Future programs schedule for later, so leave the preview running.
+ if (program.startTime <= System.currentTimeMillis()) {
+ clearPreview()
+ }
viewModelScope.launch {
val command = ScheduleRecordingCommand(
contentType = ContentType.LIVE,
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerLifecycleActions.kt b/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerLifecycleActions.kt
index 78242761..54cbb184 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerLifecycleActions.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerLifecycleActions.kt
@@ -98,6 +98,7 @@ fun PlayerViewModel.onPlayerScreenDisposed() {
}
}
playerEngine.stopLiveTimeshift()
+ stopLiveTranslationSession(clearEnabledState = false)
stopActiveStalkerPlaybackFetchDeferral()
clearPlaybackTimers()
}
@@ -149,6 +150,7 @@ fun PlayerViewModel.handOffPlaybackToMultiView() {
viewModelScope.launch { persistPlaybackProgress() }
}
playerEngine.stopLiveTimeshift()
+ stopLiveTranslationSession(clearEnabledState = false)
stopActiveStalkerPlaybackFetchDeferral()
livePreviewHandoffManager.clear(playerEngine)
}
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerLiveTranslationActions.kt b/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerLiveTranslationActions.kt
new file mode 100644
index 00000000..70c6737f
--- /dev/null
+++ b/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerLiveTranslationActions.kt
@@ -0,0 +1,76 @@
+package com.streamvault.app.ui.screens.player
+
+import androidx.lifecycle.viewModelScope
+import com.streamvault.app.player.LiveTranslationClient
+import com.streamvault.app.player.LiveTranslationSession
+import com.streamvault.domain.model.ContentType
+import com.streamvault.domain.model.ProviderType
+import kotlinx.coroutines.flow.first
+import kotlinx.coroutines.launch
+
+internal fun PlayerViewModel.toggleLiveTranslation() {
+ viewModelScope.launch {
+ val nextEnabled = !liveTranslationEnabled.value
+ preferencesRepository.setPlayerLiveTranslationEnabled(nextEnabled)
+ if (!nextEnabled) {
+ stopLiveTranslationSession()
+ } else {
+ evaluateLiveTranslationSession()
+ }
+ }
+}
+
+internal suspend fun PlayerViewModel.evaluateLiveTranslationSession() {
+ val enabled = preferencesRepository.playerLiveTranslationEnabled.first()
+ if (currentProviderId <= 0L) {
+ stopLiveTranslationSession()
+ return
+ }
+ val provider = providerRepository.getProvider(currentProviderId)
+ if (!shouldEnableLiveTranslationSession(enabled, currentContentType, provider?.type)) {
+ stopLiveTranslationSession()
+ return
+ }
+ if (currentResolvedStreamInfo == null) {
+ stopLiveTranslationSession()
+ return
+ }
+ val endpoint = preferencesRepository.playerLiveTranslationEndpoint.first()
+ val session = LiveTranslationSession(
+ scope = viewModelScope,
+ playerEngine = playerEngine,
+ client = LiveTranslationClient(okHttpClient, endpoint),
+ logicalUrl = currentStreamUrl,
+ providerId = currentProviderId,
+ contentId = currentContentId,
+ onSourceLanguageDetected = { language ->
+ _liveTranslationDetectedLanguage.value = language
+ },
+ onError = { message ->
+ showPlayerNotice(message = message, recoveryType = PlayerRecoveryType.NETWORK)
+ }
+ )
+ stopLiveTranslationSession(clearEnabledState = false)
+ liveTranslationSession = session
+ session.start()
+}
+
+internal fun PlayerViewModel.stopLiveTranslationSession(clearEnabledState: Boolean = true) {
+ liveTranslationSession?.stop()
+ liveTranslationSession = null
+ playerEngine.clearInjectedSubtitleCues()
+ _liveTranslationDetectedLanguage.value = null
+ if (clearEnabledState) {
+ _liveTranslationEnabled.value = false
+ }
+}
+
+internal fun shouldEnableLiveTranslationSession(
+ enabledPreference: Boolean,
+ contentType: ContentType,
+ providerType: ProviderType?
+): Boolean {
+ return enabledPreference &&
+ contentType == ContentType.LIVE &&
+ (providerType == ProviderType.XTREAM_CODES || providerType == ProviderType.M3U)
+}
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerViewModel.kt b/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerViewModel.kt
index 8ce5dff0..eaf05303 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerViewModel.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerViewModel.kt
@@ -10,6 +10,7 @@ import com.streamvault.app.cast.CastMediaRequest
import com.streamvault.app.cast.CastStartResult
import com.streamvault.app.di.MainPlayerEngine
import com.streamvault.app.player.LivePreviewHandoffManager
+import com.streamvault.app.player.LiveTranslationSession
import com.streamvault.app.plugins.StreamVaultPluginManager
import com.streamvault.app.util.isPlaybackComplete
import com.streamvault.app.tv.LauncherRecommendationsManager
@@ -100,7 +101,7 @@ class PlayerViewModel @Inject constructor(
internal val seekThumbnailProvider: SeekThumbnailProvider,
internal val livePreviewHandoffManager: LivePreviewHandoffManager,
internal val syncManager: SyncManager,
- private val okHttpClient: OkHttpClient,
+ internal val okHttpClient: OkHttpClient,
) : ViewModel() {
companion object {
private const val TAG = "PlayerViewModel"
@@ -340,6 +341,11 @@ class PlayerViewModel @Inject constructor(
internal var playbackTimerDefaultsApplied = false
internal var sleepTimerExitEmitted = false
internal var activeStalkerPlaybackProviderId: Long? = null
+ internal val _liveTranslationEnabled = MutableStateFlow(false)
+ val liveTranslationEnabled: StateFlow<Boolean> = _liveTranslationEnabled.asStateFlow()
+ internal val _liveTranslationDetectedLanguage = MutableStateFlow<String?>(null)
+ val liveTranslationDetectedLanguage: StateFlow<String?> = _liveTranslationDetectedLanguage.asStateFlow()
+ internal var liveTranslationSession: LiveTranslationSession? = null
val castConnectionState: StateFlow<CastConnectionState> = castManager.connectionState
@@ -514,6 +520,12 @@ class PlayerViewModel @Inject constructor(
}.combine(activePlayerEngineFlow) { style, engine -> engine to style }
.collect { (engine, style) -> engine.setSubtitleStyle(style) }
}
+ viewModelScope.launch {
+ preferencesRepository.playerLiveTranslationEnabled.collect { enabled ->
+ _liveTranslationEnabled.value = enabled
+ evaluateLiveTranslationSession()
+ }
+ }
viewModelScope.launch {
combine(
preferencesRepository.playerControlsTimeoutSeconds,
@@ -911,6 +923,7 @@ class PlayerViewModel @Inject constructor(
internal fun beginPlaybackSession(): Long {
recoveryJob?.cancel()
thumbnailPreloadJob?.cancel()
+ stopLiveTranslationSession(clearEnabledState = false)
hasRetriedXtreamAuthRefresh = false
lastRecordedVariantObservationSignature = null
readySideEffectsRequestVersion = null
@@ -1192,6 +1205,7 @@ class PlayerViewModel @Inject constructor(
currentResolvedStreamInfo = preparedStreamInfo
readySideEffectsRequestVersion = requestVersion
playerEngine.prepare(preparedStreamInfo)
+ evaluateLiveTranslationSession()
startTokenRenewalMonitoring(preparedStreamInfo.expirationTime)
return true
}
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsContentPane.kt b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsContentPane.kt
index ec2d6e9c..50a105db 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsContentPane.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsContentPane.kt
@@ -74,6 +74,7 @@ internal fun SettingsContentPane(
subtitleSizeLabel = screenLabels.subtitleSizeLabel,
subtitleTextColorLabel = screenLabels.subtitleTextColorLabel,
subtitleBackgroundLabel = screenLabels.subtitleBackgroundLabel,
+ liveTranslationEndpointLabel = screenLabels.liveTranslationEndpointLabel,
wifiQualityLabel = screenLabels.wifiQualityLabel,
ethernetQualityLabel = screenLabels.ethernetQualityLabel,
lastSpeedTestLabel = screenLabels.lastSpeedTestLabel,
@@ -96,6 +97,7 @@ internal fun SettingsContentPane(
onShowSubtitleSizeDialogChange = { dialogState.showSubtitleSizeDialog = it },
onShowSubtitleTextColorDialogChange = { dialogState.showSubtitleTextColorDialog = it },
onShowSubtitleBackgroundDialogChange = { dialogState.showSubtitleBackgroundDialog = it },
+ onShowLiveTranslationEndpointDialogChange = { dialogState.showLiveTranslationEndpointDialog = it },
onShowWifiQualityDialogChange = { dialogState.showWifiQualityDialog = it },
onShowEthernetQualityDialogChange = { dialogState.showEthernetQualityDialog = it }
)
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPlaybackSection.kt b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPlaybackSection.kt
index 60077ede..52dbd7e0 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPlaybackSection.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPlaybackSection.kt
@@ -44,6 +44,7 @@ internal fun LazyListScope.settingsPlaybackSection(
subtitleSizeLabel: String,
subtitleTextColorLabel: String,
subtitleBackgroundLabel: String,
+ liveTranslationEndpointLabel: String,
wifiQualityLabel: String,
ethernetQualityLabel: String,
lastSpeedTestLabel: String,
@@ -66,6 +67,7 @@ internal fun LazyListScope.settingsPlaybackSection(
onShowSubtitleSizeDialogChange: (Boolean) -> Unit,
onShowSubtitleTextColorDialogChange: (Boolean) -> Unit,
onShowSubtitleBackgroundDialogChange: (Boolean) -> Unit,
+ onShowLiveTranslationEndpointDialogChange: (Boolean) -> Unit,
onShowWifiQualityDialogChange: (Boolean) -> Unit,
onShowEthernetQualityDialogChange: (Boolean) -> Unit
) {
@@ -139,6 +141,34 @@ internal fun LazyListScope.settingsPlaybackSection(
}
}
HorizontalDivider(color = Color.White.copy(alpha = 0.07f), modifier = Modifier.padding(vertical = 4.dp))
+ TvClickableSurface(
+ onClick = { viewModel.setPlayerLiveTranslationEnabled(!uiState.playerLiveTranslationEnabled) },
+ shape = ClickableSurfaceDefaults.shape(RoundedCornerShape(8.dp)),
+ colors = ClickableSurfaceDefaults.colors(
+ containerColor = Color.Transparent,
+ focusedContainerColor = Primary.copy(alpha = 0.15f)
+ ),
+ scale = ClickableSurfaceDefaults.scale(focusedScale = 1f),
+ modifier = Modifier.fillMaxWidth()
+ ) {
+ Row(
+ modifier = Modifier.fillMaxWidth().padding(horizontal = 8.dp, vertical = 12.dp),
+ horizontalArrangement = Arrangement.SpaceBetween,
+ verticalAlignment = Alignment.CenterVertically
+ ) {
+ Column(modifier = Modifier.weight(1f)) {
+ Text(text = stringResource(R.string.settings_live_translation_enabled), style = MaterialTheme.typography.bodyMedium, color = OnSurface)
+ Text(text = stringResource(R.string.settings_live_translation_enabled_subtitle), style = MaterialTheme.typography.bodySmall, color = OnBackground.copy(alpha = 0.6f))
+ }
+ Switch(checked = uiState.playerLiveTranslationEnabled, onCheckedChange = { viewModel.setPlayerLiveTranslationEnabled(it) })
+ }
+ }
+ ClickableSettingsRow(
+ label = stringResource(R.string.settings_live_translation_endpoint),
+ value = liveTranslationEndpointLabel,
+ onClick = { onShowLiveTranslationEndpointDialogChange(true) }
+ )
+ HorizontalDivider(color = Color.White.copy(alpha = 0.07f), modifier = Modifier.padding(vertical = 4.dp))
TvClickableSurface(
onClick = { viewModel.setPlayerTimeshiftEnabled(!uiState.playerTimeshiftEnabled) },
shape = ClickableSurfaceDefaults.shape(RoundedCornerShape(8.dp)),
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPlayerPreferenceDialogs.kt b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPlayerPreferenceDialogs.kt
index 1d9fd149..b0f596fa 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPlayerPreferenceDialogs.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPlayerPreferenceDialogs.kt
@@ -43,7 +43,9 @@ internal fun SettingsPlayerPreferenceDialogs(
showNoticeTimeoutDialog: Boolean,
onShowNoticeTimeoutDialogChange: (Boolean) -> Unit,
showDiagnosticsTimeoutDialog: Boolean,
- onShowDiagnosticsTimeoutDialogChange: (Boolean) -> Unit
+ onShowDiagnosticsTimeoutDialogChange: (Boolean) -> Unit,
+ showLiveTranslationEndpointDialog: Boolean,
+ onShowLiveTranslationEndpointDialogChange: (Boolean) -> Unit
) {
if (showPlaybackSpeedDialog) {
val speedOptions = remember { listOf(0.5f, 0.75f, 1f, 1.25f, 1.5f, 2f) }
@@ -298,6 +300,25 @@ internal fun SettingsPlayerPreferenceDialogs(
}
)
}
+
+ if (showLiveTranslationEndpointDialog) {
+ SimpleTextValueDialog(
+ title = stringResource(R.string.settings_live_translation_endpoint),
+ subtitle = stringResource(R.string.settings_live_translation_endpoint_subtitle),
+ initialValue = uiState.playerLiveTranslationEndpoint,
+ onDismiss = { onShowLiveTranslationEndpointDialogChange(false) },
+ onConfirm = { endpoint ->
+ if (endpoint.isBlank()) {
+ viewModel.showUserMessage(
+ context.getString(R.string.settings_live_translation_endpoint_empty_error)
+ )
+ } else {
+ viewModel.setPlayerLiveTranslationEndpoint(endpoint)
+ }
+ onShowLiveTranslationEndpointDialogChange(false)
+ }
+ )
+ }
}
private fun AppTimeFormat.labelResId(): Int = when (this) {
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPreferenceDialogs.kt b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPreferenceDialogs.kt
index db8910fa..24437bba 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPreferenceDialogs.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPreferenceDialogs.kt
@@ -56,6 +56,8 @@ internal fun SettingsPreferenceDialogs(
onShowSubtitleTextColorDialogChange: (Boolean) -> Unit,
showSubtitleBackgroundDialog: Boolean,
onShowSubtitleBackgroundDialogChange: (Boolean) -> Unit,
+ showLiveTranslationEndpointDialog: Boolean,
+ onShowLiveTranslationEndpointDialogChange: (Boolean) -> Unit,
showWifiQualityDialog: Boolean,
onShowWifiQualityDialogChange: (Boolean) -> Unit,
showEthernetQualityDialog: Boolean,
@@ -96,7 +98,9 @@ internal fun SettingsPreferenceDialogs(
showNoticeTimeoutDialog = showNoticeTimeoutDialog,
onShowNoticeTimeoutDialogChange = onShowNoticeTimeoutDialogChange,
showDiagnosticsTimeoutDialog = showDiagnosticsTimeoutDialog,
- onShowDiagnosticsTimeoutDialogChange = onShowDiagnosticsTimeoutDialogChange
+ onShowDiagnosticsTimeoutDialogChange = onShowDiagnosticsTimeoutDialogChange,
+ showLiveTranslationEndpointDialog = showLiveTranslationEndpointDialog,
+ onShowLiveTranslationEndpointDialogChange = onShowLiveTranslationEndpointDialogChange
)
if (showGuideDefaultCategoryDialog) {
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPreferenceModels.kt b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPreferenceModels.kt
index 9cb23a3e..a823cb32 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPreferenceModels.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPreferenceModels.kt
@@ -57,6 +57,8 @@ internal data class SettingsPreferenceSnapshot(
val subtitleTextScale: Float,
val subtitleTextColor: Int,
val subtitleBackgroundColor: Int,
+ val playerLiveTranslationEnabled: Boolean,
+ val playerLiveTranslationEndpoint: String,
val wifiMaxVideoHeight: Int?,
val ethernetMaxVideoHeight: Int?,
val playerTimeshiftEnabled: Boolean,
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPreferenceSnapshotMapper.kt b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPreferenceSnapshotMapper.kt
index 0babe802..af835533 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPreferenceSnapshotMapper.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsPreferenceSnapshotMapper.kt
@@ -27,6 +27,8 @@ internal fun SettingsUiState.applyPreferenceSnapshot(snapshot: SettingsPreferenc
subtitleTextScale = snapshot.subtitleTextScale,
subtitleTextColor = snapshot.subtitleTextColor,
subtitleBackgroundColor = snapshot.subtitleBackgroundColor,
+ playerLiveTranslationEnabled = snapshot.playerLiveTranslationEnabled,
+ playerLiveTranslationEndpoint = snapshot.playerLiveTranslationEndpoint,
wifiMaxVideoHeight = snapshot.wifiMaxVideoHeight,
ethernetMaxVideoHeight = snapshot.ethernetMaxVideoHeight,
playerTimeshiftEnabled = snapshot.playerTimeshiftEnabled,
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsScreenDialogState.kt b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsScreenDialogState.kt
index 1450c088..63a59fa6 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsScreenDialogState.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsScreenDialogState.kt
@@ -39,6 +39,7 @@ internal class SettingsScreenDialogState(
private val showSubtitleSizeDialogState: MutableState<Boolean>,
private val showSubtitleTextColorDialogState: MutableState<Boolean>,
private val showSubtitleBackgroundDialogState: MutableState<Boolean>,
+ private val showLiveTranslationEndpointDialogState: MutableState<Boolean>,
private val showWifiQualityDialogState: MutableState<Boolean>,
private val showProviderSyncDialogState: MutableState<Boolean>,
private val showCustomProviderSyncDialogState: MutableState<Boolean>,
@@ -93,6 +94,7 @@ internal class SettingsScreenDialogState(
var showSubtitleSizeDialog by showSubtitleSizeDialogState
var showSubtitleTextColorDialog by showSubtitleTextColorDialogState
var showSubtitleBackgroundDialog by showSubtitleBackgroundDialogState
+ var showLiveTranslationEndpointDialog by showLiveTranslationEndpointDialogState
var showWifiQualityDialog by showWifiQualityDialogState
var showProviderSyncDialog by showProviderSyncDialogState
var showCustomProviderSyncDialog by showCustomProviderSyncDialogState
@@ -215,6 +217,7 @@ internal fun rememberSettingsScreenDialogState(): SettingsScreenDialogState {
val showSubtitleSizeDialogState = rememberSaveable { mutableStateOf(false) }
val showSubtitleTextColorDialogState = rememberSaveable { mutableStateOf(false) }
val showSubtitleBackgroundDialogState = rememberSaveable { mutableStateOf(false) }
+ val showLiveTranslationEndpointDialogState = rememberSaveable { mutableStateOf(false) }
val showWifiQualityDialogState = rememberSaveable { mutableStateOf(false) }
val showProviderSyncDialogState = rememberSaveable { mutableStateOf(false) }
val showCustomProviderSyncDialogState = rememberSaveable { mutableStateOf(false) }
@@ -278,6 +281,7 @@ internal fun rememberSettingsScreenDialogState(): SettingsScreenDialogState {
showSubtitleSizeDialogState = showSubtitleSizeDialogState,
showSubtitleTextColorDialogState = showSubtitleTextColorDialogState,
showSubtitleBackgroundDialogState = showSubtitleBackgroundDialogState,
+ showLiveTranslationEndpointDialogState = showLiveTranslationEndpointDialogState,
showWifiQualityDialogState = showWifiQualityDialogState,
showProviderSyncDialogState = showProviderSyncDialogState,
showCustomProviderSyncDialogState = showCustomProviderSyncDialogState,
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsScreenDialogs.kt b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsScreenDialogs.kt
index e5eca30b..92960a93 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsScreenDialogs.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsScreenDialogs.kt
@@ -155,6 +155,8 @@ internal fun SettingsScreenDialogs(
onShowSubtitleTextColorDialogChange = { dialogState.showSubtitleTextColorDialog = it },
showSubtitleBackgroundDialog = dialogState.showSubtitleBackgroundDialog,
onShowSubtitleBackgroundDialogChange = { dialogState.showSubtitleBackgroundDialog = it },
+ showLiveTranslationEndpointDialog = dialogState.showLiveTranslationEndpointDialog,
+ onShowLiveTranslationEndpointDialogChange = { dialogState.showLiveTranslationEndpointDialog = it },
showWifiQualityDialog = dialogState.showWifiQualityDialog,
onShowWifiQualityDialogChange = { dialogState.showWifiQualityDialog = it },
showEthernetQualityDialog = dialogState.showEthernetQualityDialog,
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsScreenState.kt b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsScreenState.kt
index 38e1dd7a..7ec749f6 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsScreenState.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsScreenState.kt
@@ -30,6 +30,7 @@ internal data class SettingsScreenLabels(
val subtitleSizeLabel: String,
val subtitleTextColorLabel: String,
val subtitleBackgroundLabel: String,
+ val liveTranslationEndpointLabel: String,
val wifiQualityLabel: String,
val ethernetQualityLabel: String,
val timeshiftDepthLabel: String,
@@ -100,6 +101,9 @@ internal fun rememberSettingsScreenLabels(
val subtitleBackgroundLabel = remember(uiState.subtitleBackgroundColor, context) {
formatSubtitleColorLabel(uiState.subtitleBackgroundColor, subtitleBackgroundColorOptions(context))
}
+ val liveTranslationEndpointLabel = remember(uiState.playerLiveTranslationEndpoint) {
+ uiState.playerLiveTranslationEndpoint
+ }
val wifiQualityLabel = remember(uiState.wifiMaxVideoHeight, context) {
formatQualityCapLabel(uiState.wifiMaxVideoHeight, context.getString(R.string.settings_quality_cap_auto))
}
@@ -167,6 +171,7 @@ internal fun rememberSettingsScreenLabels(
subtitleSizeLabel = subtitleSizeLabel,
subtitleTextColorLabel = subtitleTextColorLabel,
subtitleBackgroundLabel = subtitleBackgroundLabel,
+ liveTranslationEndpointLabel = liveTranslationEndpointLabel,
wifiQualityLabel = wifiQualityLabel,
ethernetQualityLabel = ethernetQualityLabel,
timeshiftDepthLabel = timeshiftDepthLabel,
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsStateBindings.kt b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsStateBindings.kt
index 5011088f..b3839237 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsStateBindings.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsStateBindings.kt
@@ -56,6 +56,8 @@ internal fun observeSettingsPreferenceSnapshot(
subtitleTextScale = 1f,
subtitleTextColor = 0xFFFFFFFF.toInt(),
subtitleBackgroundColor = 0x80000000.toInt(),
+ playerLiveTranslationEnabled = false,
+ playerLiveTranslationEndpoint = "http://10.0.2.2:8177",
wifiMaxVideoHeight = null,
ethernetMaxVideoHeight = null,
playerTimeshiftEnabled = false,
@@ -137,6 +139,10 @@ internal fun observeSettingsPreferenceSnapshot(
snapshot.copy(subtitleTextColor = subtitleTextColor)
}.combine(preferencesRepository.playerSubtitleBackgroundColor) { snapshot, subtitleBackgroundColor ->
snapshot.copy(subtitleBackgroundColor = subtitleBackgroundColor)
+ }.combine(preferencesRepository.playerLiveTranslationEnabled) { snapshot, enabled ->
+ snapshot.copy(playerLiveTranslationEnabled = enabled)
+ }.combine(preferencesRepository.playerLiveTranslationEndpoint) { snapshot, endpoint ->
+ snapshot.copy(playerLiveTranslationEndpoint = endpoint)
}.combine(preferencesRepository.playerWifiMaxVideoHeight) { snapshot, wifiMaxVideoHeight ->
snapshot.copy(wifiMaxVideoHeight = wifiMaxVideoHeight)
}.combine(preferencesRepository.playerEthernetMaxVideoHeight) { snapshot, ethernetMaxVideoHeight ->
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsUiStateModel.kt b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsUiStateModel.kt
index 723670fe..54601c5f 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsUiStateModel.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsUiStateModel.kt
@@ -76,6 +76,8 @@ data class SettingsUiState(
val subtitleTextScale: Float = 1f,
val subtitleTextColor: Int = 0xFFFFFFFF.toInt(),
val subtitleBackgroundColor: Int = 0x80000000.toInt(),
+ val playerLiveTranslationEnabled: Boolean = false,
+ val playerLiveTranslationEndpoint: String = "http://10.0.2.2:8177",
val wifiMaxVideoHeight: Int? = null,
val ethernetMaxVideoHeight: Int? = null,
val playerTimeshiftEnabled: Boolean = false,
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsViewModel.kt b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsViewModel.kt
index 7d6ab753..0a63d147 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsViewModel.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/settings/SettingsViewModel.kt
@@ -633,6 +633,18 @@ class SettingsViewModel @Inject constructor(
}
}
+ fun setPlayerLiveTranslationEnabled(enabled: Boolean) {
+ viewModelScope.launch {
+ preferencesRepository.setPlayerLiveTranslationEnabled(enabled)
+ }
+ }
+
+ fun setPlayerLiveTranslationEndpoint(endpoint: String) {
+ viewModelScope.launch {
+ preferencesRepository.setPlayerLiveTranslationEndpoint(endpoint)
+ }
+ }
+
fun setPlayerTimeshiftDepthMinutes(minutes: Int) {
viewModelScope.launch {
preferencesRepository.setPlayerTimeshiftDepthMinutes(minutes)
diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml
index ffad41db..3cebcc83 100644
--- a/app/src/main/res/values/strings.xml
+++ b/app/src/main/res/values/strings.xml
@@ -404,6 +404,11 @@
<string name="settings_media_session_subtitle">Expose playback to Android system media controls. Disable this if playback crashes on a specific device when opening the player.</string>
<string name="settings_live_timeshift">Enable Local Live Rewind</string>
<string name="settings_live_timeshift_subtitle">Keep a temporary local buffer for pausing and rewinding live TV even when the provider has no catch-up support.</string>
+ <string name="settings_live_translation_enabled">Enable Live Translation Subtitles</string>
+ <string name="settings_live_translation_enabled_subtitle">Translate spoken live-channel audio into English subtitles using your local Whisper service.</string>
+ <string name="settings_live_translation_endpoint">Live Translation Service Endpoint</string>
+ <string name="settings_live_translation_endpoint_subtitle">Base URL for the local translation service (emulator default: http://10.0.2.2:8177).</string>
+ <string name="settings_live_translation_endpoint_empty_error">Live translation endpoint cannot be empty.</string>
<string name="settings_live_timeshift_depth">Local Live Rewind Depth</string>
<string name="settings_live_timeshift_backend">Live Rewind Backend</string>
<string name="settings_live_timeshift_backend_value">Automatic</string>
diff --git a/app/src/test/java/com/streamvault/app/player/LiveTranslationClientTest.kt b/app/src/test/java/com/streamvault/app/player/LiveTranslationClientTest.kt
new file mode 100644
index 00000000..cf148cbe
--- /dev/null
+++ b/app/src/test/java/com/streamvault/app/player/LiveTranslationClientTest.kt
@@ -0,0 +1,49 @@
+package com.streamvault.app.player
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertFalse
+import kotlin.test.assertNull
+import kotlin.test.assertTrue
+
+class LiveTranslationClientTest {
+
+ @Test
+ fun parseTranslationUpdate_mapsFinalUpdate() {
+ val payload = """
+ {
+ "sessionId": "s1",
+ "chunkId": 7,
+ "isFinal": true,
+ "text": "hello world",
+ "sourceLanguage": "es"
+ }
+ """.trimIndent()
+
+ val update = parseTranslationUpdate(payload)
+
+ assertEquals(7L, update.chunkId)
+ assertTrue(update.isFinal)
+ assertEquals("hello world", update.text)
+ assertEquals("es", update.sourceLanguage)
+ }
+
+ @Test
+ fun parseTranslationUpdate_handlesEmptyAndPartial() {
+ val payload = """
+ {
+ "sessionId": "s1",
+ "chunkId": 8,
+ "isFinal": false,
+ "text": " "
+ }
+ """.trimIndent()
+
+ val update = parseTranslationUpdate(payload)
+
+ assertEquals(8L, update.chunkId)
+ assertFalse(update.isFinal)
+ assertEquals("", update.text)
+ assertNull(update.sourceLanguage)
+ }
+}
diff --git a/app/src/test/java/com/streamvault/app/ui/screens/player/PlayerLiveTranslationActionsTest.kt b/app/src/test/java/com/streamvault/app/ui/screens/player/PlayerLiveTranslationActionsTest.kt
new file mode 100644
index 00000000..cf307088
--- /dev/null
+++ b/app/src/test/java/com/streamvault/app/ui/screens/player/PlayerLiveTranslationActionsTest.kt
@@ -0,0 +1,53 @@
+package com.streamvault.app.ui.screens.player
+
+import com.streamvault.domain.model.ContentType
+import com.streamvault.domain.model.ProviderType
+import kotlin.test.Test
+import kotlin.test.assertFalse
+import kotlin.test.assertTrue
+
+class PlayerLiveTranslationActionsTest {
+
+ @Test
+ fun shouldEnableLiveTranslationSession_requiresEnabledSupportedProviderAndLiveContent() {
+ assertTrue(
+ shouldEnableLiveTranslationSession(
+ enabledPreference = true,
+ contentType = ContentType.LIVE,
+ providerType = ProviderType.XTREAM_CODES
+ )
+ )
+
+ assertFalse(
+ shouldEnableLiveTranslationSession(
+ enabledPreference = false,
+ contentType = ContentType.LIVE,
+ providerType = ProviderType.XTREAM_CODES
+ )
+ )
+
+ assertFalse(
+ shouldEnableLiveTranslationSession(
+ enabledPreference = true,
+ contentType = ContentType.MOVIE,
+ providerType = ProviderType.XTREAM_CODES
+ )
+ )
+
+ assertFalse(
+ shouldEnableLiveTranslationSession(
+ enabledPreference = true,
+ contentType = ContentType.LIVE,
+ providerType = ProviderType.STALKER_PORTAL
+ )
+ )
+
+ assertTrue(
+ shouldEnableLiveTranslationSession(
+ enabledPreference = true,
+ contentType = ContentType.LIVE,
+ providerType = ProviderType.M3U
+ )
+ )
+ }
+}
diff --git a/data/src/main/java/com/streamvault/data/preferences/PreferencesRepository.kt b/data/src/main/java/com/streamvault/data/preferences/PreferencesRepository.kt
index 019dfe19..191228c1 100644
--- a/data/src/main/java/com/streamvault/data/preferences/PreferencesRepository.kt
+++ b/data/src/main/java/com/streamvault/data/preferences/PreferencesRepository.kt
@@ -131,6 +131,8 @@ class PreferencesRepository @Inject constructor(
val PLAYER_SUBTITLE_TEXT_SCALE = stringPreferencesKey("player_subtitle_text_scale")
val PLAYER_SUBTITLE_TEXT_COLOR = intPreferencesKey("player_subtitle_text_color")
val PLAYER_SUBTITLE_BACKGROUND_COLOR = intPreferencesKey("player_subtitle_background_color")
+ val PLAYER_LIVE_TRANSLATION_ENABLED = booleanPreferencesKey("player_live_translation_enabled")
+ val PLAYER_LIVE_TRANSLATION_ENDPOINT = stringPreferencesKey("player_live_translation_endpoint")
val PLAYER_CONTROLS_TIMEOUT_SECONDS = intPreferencesKey("player_controls_timeout_seconds")
val PLAYER_LIVE_OVERLAY_TIMEOUT_SECONDS = intPreferencesKey("player_live_overlay_timeout_seconds")
val PLAYER_NOTICE_TIMEOUT_SECONDS = intPreferencesKey("player_notice_timeout_seconds")
@@ -337,6 +339,17 @@ class PreferencesRepository @Inject constructor(
preferences[PreferencesKeys.PLAYER_SUBTITLE_BACKGROUND_COLOR] ?: 0x80000000.toInt()
}
+ val playerLiveTranslationEnabled: Flow<Boolean> = context.dataStore.data.map { preferences ->
+ preferences[PreferencesKeys.PLAYER_LIVE_TRANSLATION_ENABLED] ?: false
+ }
+
+ val playerLiveTranslationEndpoint: Flow<String> = context.dataStore.data.map { preferences ->
+ preferences[PreferencesKeys.PLAYER_LIVE_TRANSLATION_ENDPOINT]
+ ?.trim()
+ ?.takeIf { it.isNotBlank() }
+ ?: "http://10.0.2.2:8177"
+ }
+
val playerControlsTimeoutSeconds: Flow<Int> = context.dataStore.data.map { preferences ->
(preferences[PreferencesKeys.PLAYER_CONTROLS_TIMEOUT_SECONDS] ?: 5).coerceIn(2, 60)
}
@@ -863,6 +876,23 @@ class PreferencesRepository @Inject constructor(
}
}
+ suspend fun setPlayerLiveTranslationEnabled(enabled: Boolean) {
+ context.dataStore.edit { preferences ->
+ preferences[PreferencesKeys.PLAYER_LIVE_TRANSLATION_ENABLED] = enabled
+ }
+ }
+
+ suspend fun setPlayerLiveTranslationEndpoint(endpoint: String) {
+ context.dataStore.edit { preferences ->
+ val normalized = endpoint.trim()
+ if (normalized.isBlank()) {
+ preferences.remove(PreferencesKeys.PLAYER_LIVE_TRANSLATION_ENDPOINT)
+ } else {
+ preferences[PreferencesKeys.PLAYER_LIVE_TRANSLATION_ENDPOINT] = normalized
+ }
+ }
+ }
+
suspend fun setPlayerControlsTimeoutSeconds(seconds: Int) {
context.dataStore.edit { preferences ->
preferences[PreferencesKeys.PLAYER_CONTROLS_TIMEOUT_SECONDS] = seconds.coerceIn(2, 60)
diff --git a/docs/LOCAL_LIVE_TRANSLATION.md b/docs/LOCAL_LIVE_TRANSLATION.md
new file mode 100644
index 00000000..4ef68331
--- /dev/null
+++ b/docs/LOCAL_LIVE_TRANSLATION.md
@@ -0,0 +1,160 @@
+# Local Live Translation (Whisper)
+
+This feature adds optional live translation subtitles for `LIVE` playback in the player.
+
+## Current app scope
+
+- Supported for `LIVE` playback from `Xtream` and `M3U` providers.
+- The player sends session metadata and tapped decoded PCM audio to a local translation service.
+- The service is expected to run on the same Mac as the Android emulator (`http://10.0.2.2:8177` default).
+- Source language is expected to be auto-detected by the service.
+- Returned text is rendered as a single caption line, one phrase at a time
+ (replaced in place), and cleared after a short idle/linger period.
+- The app paces caption changes for readability: a line is held for at least
+ `MIN_CAPTION_DISPLAY_MS` (~2.2s) before advancing to the latest text (intermediate
+ updates are coalesced), trading a little latency so text doesn't change too fast.
+- Control lives in Settings:
+ - global Settings switch + endpoint
+
+## Streaming model (phrase + VAD)
+
+Adapted from [Vanyoo/realtime-subtitle](https://github.com/Vanyoo/realtime-subtitle).
+The service accumulates audio into a phrase and uses simple RMS VAD to decide when
+to emit text, keyed by an incrementing `chunkId`:
+
+- The app uploads short audio increments (~1s) tapped from the decoded player audio.
+- The service appends them into a per-session phrase buffer and, on each upload:
+ - emits an in-progress **partial** (same `chunkId`) at most every
+ `WHISPER_UPDATE_INTERVAL` so text appears while speech is ongoing;
+ - **finalises** the phrase (`isFinal=true`, then resets the buffer and bumps
+ `chunkId`) on a natural pause — a silence tail past ~2s, a short pause past
+ `WHISPER_SOFT_LIMIT_SECONDS`, or a hard cap at `WHISPER_MAX_PHRASE_SECONDS`.
+- Because each phrase is one `chunkId` and the buffer resets on finalise, text is
+ never re-emitted — the app shows one line, replacing it in place. Partials of the
+ same phrase grow the line; a new `chunkId` replaces it with the next phrase.
+- No `initial_prompt` is used: with `task=translate`, feeding the previous
+ translation back makes Whisper re-emit it at the start of the next phrase
+ (re-showing already-displayed text). Hallucinations (repetition loops, silence)
+ are still filtered server-side.
+
+## Service API contract
+
+The Android app uses these endpoints:
+
+1. `POST /v1/live-translation/session`
+ - Body:
+ - `logicalStreamUrl`
+ - `providerId`
+ - `contentId`
+ - `targetLanguage` (`"en"`)
+ - `task` (`"translate"`)
+ - `model` (`"whisper-large-v3"`)
+ - `autoDetectSourceLanguage` (`true`)
+ - Response:
+ - `sessionId` (or `id`)
+
+2. `POST /v1/live-translation/session/{sessionId}/audio?startMs=<long>&endMs=<long>&sampleRate=16000&channelCount=1&encoding=pcm_s16le`
+ - Body:
+ - raw 16-bit little-endian mono PCM at 16 kHz (sent as ~1s increments)
+ - Response (`TranslationUpdate`):
+ - `sessionId`
+ - `chunkId` (phrase id; partials share it, finalise bumps it)
+ - `isFinal` (true once the phrase is finalised)
+ - `text` (English translation so far; blank when silence / no new text)
+ - `sourceLanguage` (optional, auto-detected)
+
+3. `DELETE /v1/live-translation/session/{sessionId}`
+ - Best-effort session teardown.
+
+## Whisper recommendation
+
+To keep UI language selection simple, run Whisper in translation mode with auto language detection:
+
+- `task=translate`
+- do not force `language`
+- use `whisper-large-v3` (app sends this in session start)
+
+This yields English output subtitles from multilingual speech inputs.
+
+## Quick local startup (ready-to-run)
+
+Start the included local service with model `large-v3`:
+
+```bash
+HF_HOME="$(pwd)/tools/live-translation-service/.hf-cache" \
+WHISPER_MODEL=large-v3 \
+tools/live-translation-service/.venv/bin/uvicorn service:app \
+ --app-dir tools/live-translation-service \
+ --host 0.0.0.0 \
+ --port 8177
+```
+
+### Compute resources (Apple Silicon)
+
+- **CPU backend (default, `faster-whisper`)**: CTranslate2 is **CPU-only on macOS** — it
+ cannot use the Mac GPU/Neural Engine. The service auto-detects and uses all
+ performance cores (`hw.perflevel0.logicalcpu`, e.g. 20 on an M3 Ultra). Override
+ with `WHISPER_CPU_THREADS`, `WHISPER_NUM_WORKERS`, `WHISPER_COMPUTE_TYPE` (default
+ `int8`, the fastest on CPU). The model is warmed on startup to avoid first-call lag.
+- **GPU backend (`mlx`)**: to actually use the M3 GPU (Metal), install MLX and switch
+ backends:
+
+ ```bash
+ tools/live-translation-service/.venv/bin/pip install mlx-whisper
+ # then start the service with:
+ WHISPER_BACKEND=mlx WHISPER_MLX_MODEL=mlx-community/whisper-large-v3-mlx ...
+ ```
+
+ This is typically several times faster than CPU for `large-v3` on Apple Silicon.
+
+`GET /health` reports the active `backend`, `cpuThreads`, `numWorkers`, `computeType`,
+and `translationMode`/`llmModel`/`llmReady`.
+
+### Translation mode: Whisper vs. local LLM
+
+There are two ways to produce the English text:
+
+- **`whisper` (default)** — Whisper itself translates (`task=translate`). One model,
+ lowest latency, but it can occasionally re-emit already-shown text and cannot be
+ *instructed* to avoid that.
+- **`llm`** — Whisper transcribes the **source language** (`task=transcribe`), then a
+ local OpenAI-compatible LLM (LM Studio / Ollama) translates each phrase into
+ English. Partials (in-progress) are translated without prior context for low
+ latency; finals use prior phrase context and are told not to repeat. HTTP uploads
+ return immediately while Whisper+LLM run in a background worker per session, so
+ audio keeps flowing. Default phrase window is shorter in LLM mode (6 s hard / 4 s
+ soft) than whisper mode (12 s / 6 s).
+
+Enable LLM mode (LM Studio running on `:1234`):
+
+```bash
+tools/live-translation-service/.venv/bin/pip install openai httpx
+HF_HOME="$(pwd)/tools/live-translation-service/.hf-cache" \
+WHISPER_MODEL=large-v3 WHISPER_BACKEND=mlx \
+WHISPER_TRANSLATION_MODE=llm \
+LLM_BASE_URL=http://localhost:1234/v1 \
+LLM_MODEL=qwen3.6-35b-a3b \
+tools/live-translation-service/.venv/bin/uvicorn service:app \
+ --app-dir tools/live-translation-service --host 0.0.0.0 --port 8177
+```
+
+LLM env vars: `LLM_BASE_URL`, `LLM_API_KEY` (default `lm-studio`), `LLM_MODEL`,
+`LLM_TARGET_LANG` (default `English`), `LLM_NO_THINK` (default `true` — appends
+`/no_think` and strips `<think>` blocks for reasoning models like Qwen3). The Android
+app and PCM upload contract are unchanged; only the service config differs.
+
+Tunable phrase/VAD env vars (all optional):
+
+- `WHISPER_SILENCE_RMS` (default `0.01`) — silence amplitude threshold.
+- `WHISPER_SILENCE_DURATION` (default `0.8`) — silence tail that finalises a phrase.
+- `WHISPER_SOFT_LIMIT_SECONDS` (default `6`) / `WHISPER_MAX_PHRASE_SECONDS` (default `12`).
+- `WHISPER_UPDATE_INTERVAL` (default `0.8`) — min spacing between partials.
+- `WHISPER_LANGUAGE_PIN_MIN_PROB` (default `0.7`) — confidence to lock source language.
+
+Equivalent docker/env setups are fine as long as your API accepts the same PCM upload contract.
+
+## Emulator networking reminder
+
+- Android emulator cannot reach host `localhost` directly for host-machine services.
+- Use `10.0.2.2` for services running on your Mac.
+- Default expected endpoint in app preferences: `http://10.0.2.2:8177`.
diff --git a/player/src/main/java/com/streamvault/player/LiveAudioTap.kt b/player/src/main/java/com/streamvault/player/LiveAudioTap.kt
new file mode 100644
index 00000000..80f3e89b
--- /dev/null
+++ b/player/src/main/java/com/streamvault/player/LiveAudioTap.kt
@@ -0,0 +1,13 @@
+package com.streamvault.player
+
+data class LiveAudioPcmBuffer(
+ val data: ByteArray,
+ val presentationTimeUs: Long,
+ val sampleRate: Int,
+ val channelCount: Int,
+ val encoding: Int
+)
+
+fun interface LiveAudioTap {
+ fun onPcmAudio(buffer: LiveAudioPcmBuffer)
+}
diff --git a/player/src/main/java/com/streamvault/player/Media3PlayerEngine.kt b/player/src/main/java/com/streamvault/player/Media3PlayerEngine.kt
index ddc9d3dd..3c05978d 100644
--- a/player/src/main/java/com/streamvault/player/Media3PlayerEngine.kt
+++ b/player/src/main/java/com/streamvault/player/Media3PlayerEngine.kt
@@ -12,6 +12,7 @@ import androidx.media3.common.Format
import androidx.media3.common.PlaybackException
import androidx.media3.common.PlaybackParameters
import androidx.media3.common.Player
+import androidx.media3.common.text.Cue
import androidx.media3.common.util.UnstableApi
import androidx.media3.exoplayer.DecoderReuseEvaluation
import androidx.media3.exoplayer.DefaultLivePlaybackSpeedControl
@@ -43,6 +44,7 @@ import com.streamvault.player.playback.AutomaticRecoveryAction
import com.streamvault.player.playback.DefaultDecoderPreferencePolicy
import com.streamvault.player.playback.DefaultPlaybackCompatibilityProfile
import com.streamvault.player.playback.AudioVideoOffsetAudioSink
+import com.streamvault.player.playback.LiveAudioTapAudioSink
import com.streamvault.player.playback.PlaybackCodecSelector
import com.streamvault.player.playback.PlaybackCompatibilityProfile
import com.streamvault.player.playback.PlaybackBufferPolicies
@@ -175,6 +177,8 @@ class Media3PlayerEngine @Inject constructor(
private var videoStallSafeRecoveryPerformed = false
private var textureViewSessionFallbackAttempted = false
private var audioVideoSyncSinkActive = false
+ @Volatile
+ private var liveAudioTap: LiveAudioTap? = null
private var lastStreamInfo: StreamInfo? = null
private var lastMediaId: String? = null
private var currentResolvedStreamType: ResolvedStreamType = ResolvedStreamType.UNKNOWN
@@ -399,6 +403,7 @@ class Media3PlayerEngine @Inject constructor(
_playbackState.value = PlaybackState.IDLE
_isPlaying.value = false
_mediaTitle.value = null
+ clearInjectedSubtitleCues()
statsCollector.stop()
statsCollector.reset()
audioFocusController.onPauseOrStop()
@@ -642,6 +647,18 @@ class Media3PlayerEngine @Inject constructor(
exoPlayer?.let { trackController.selectSubtitleTrack(it, trackId) }
}
+ override fun setInjectedSubtitleCues(cues: List<Cue>) {
+ viewBinder.setInjectedSubtitleCues(cues)
+ }
+
+ override fun clearInjectedSubtitleCues() {
+ viewBinder.clearInjectedSubtitleCues()
+ }
+
+ override fun setLiveAudioTap(tap: LiveAudioTap?) {
+ liveAudioTap = tap
+ }
+
override fun addExternalSubtitle(subtitleUri: android.net.Uri, language: String) {
val player = exoPlayer ?: return
val streamInfo = lastStreamInfo ?: return
@@ -768,6 +785,7 @@ class Media3PlayerEngine @Inject constructor(
_playbackState.value = PlaybackState.IDLE
_isPlaying.value = false
_mediaTitle.value = null
+ clearInjectedSubtitleCues()
trackController.resetSelections()
statsCollector.reset()
videoStallDetector.reset()
@@ -1031,32 +1049,34 @@ class Media3PlayerEngine @Inject constructor(
)
audioVideoSyncSinkActive = false
- val factory = if (rendererPlan.useStockRenderersFactory) {
- DefaultRenderersFactory(context)
- } else {
- object : DefaultRenderersFactory(context) {
- override fun buildAudioSink(
- context: Context,
- enableFloatOutput: Boolean,
- enableAudioTrackPlaybackParams: Boolean
- ): AudioSink {
- val audioSink = requireNotNull(
- super.buildAudioSink(context, enableFloatOutput, enableAudioTrackPlaybackParams)
- ) {
- "DefaultRenderersFactory did not provide an AudioSink."
- }
- if (!rendererPlan.useAudioVideoSyncSink) {
- audioVideoSyncSinkActive = false
- return audioSink
- }
+ val factory = object : DefaultRenderersFactory(context) {
+ override fun buildAudioSink(
+ context: Context,
+ enableFloatOutput: Boolean,
+ enableAudioTrackPlaybackParams: Boolean
+ ): AudioSink {
+ val audioSink = requireNotNull(
+ super.buildAudioSink(context, enableFloatOutput, enableAudioTrackPlaybackParams)
+ ) {
+ "DefaultRenderersFactory did not provide an AudioSink."
+ }
+ val syncAwareSink = if (rendererPlan.useAudioVideoSyncSink) {
audioVideoSyncSinkActive = true
- return AudioVideoOffsetAudioSink(
+ AudioVideoOffsetAudioSink(
delegate = audioSink,
offsetUsProvider = { audioVideoOffsetUs.get() }
)
+ } else {
+ audioVideoSyncSinkActive = false
+ audioSink
}
+ return LiveAudioTapAudioSink(
+ delegate = syncAwareSink,
+ tapProvider = { liveAudioTap }
+ )
+ }
- override fun buildVideoRenderers(
+ override fun buildVideoRenderers(
context: Context,
extensionRendererMode: Int,
mediaCodecSelector: MediaCodecSelector,
@@ -1065,45 +1085,44 @@ class Media3PlayerEngine @Inject constructor(
eventListener: VideoRendererEventListener,
allowedVideoJoiningTimeMs: Long,
out: ArrayList<Renderer>
- ) {
- if (!rendererPlan.useVideoRendererWorkaround) {
- super.buildVideoRenderers(
- context,
- extensionRendererMode,
- mediaCodecSelector,
- enableDecoderFallback,
- eventHandler,
- eventListener,
- allowedVideoJoiningTimeMs,
- out
- )
- return
- }
-
- out.add(object : MediaCodecVideoRenderer(
+ ) {
+ if (!rendererPlan.useVideoRendererWorkaround) {
+ super.buildVideoRenderers(
context,
+ extensionRendererMode,
mediaCodecSelector,
- allowedVideoJoiningTimeMs,
enableDecoderFallback,
eventHandler,
eventListener,
- 50
- ) {
- override fun canReuseCodec(
- codecInfo: MediaCodecInfo,
- oldFormat: Format,
- newFormat: Format
- ): DecoderReuseEvaluation {
- return DecoderReuseEvaluation(
- codecInfo.name,
- oldFormat,
- newFormat,
- DecoderReuseEvaluation.REUSE_RESULT_NO,
- DecoderReuseEvaluation.DISCARD_REASON_MAX_INPUT_SIZE_EXCEEDED
- )
- }
- })
+ allowedVideoJoiningTimeMs,
+ out
+ )
+ return
}
+
+ out.add(object : MediaCodecVideoRenderer(
+ context,
+ mediaCodecSelector,
+ allowedVideoJoiningTimeMs,
+ enableDecoderFallback,
+ eventHandler,
+ eventListener,
+ 50
+ ) {
+ override fun canReuseCodec(
+ codecInfo: MediaCodecInfo,
+ oldFormat: Format,
+ newFormat: Format
+ ): DecoderReuseEvaluation {
+ return DecoderReuseEvaluation(
+ codecInfo.name,
+ oldFormat,
+ newFormat,
+ DecoderReuseEvaluation.REUSE_RESULT_NO,
+ DecoderReuseEvaluation.DISCARD_REASON_MAX_INPUT_SIZE_EXCEEDED
+ )
+ }
+ })
}
}
diff --git a/player/src/main/java/com/streamvault/player/PlayerEngine.kt b/player/src/main/java/com/streamvault/player/PlayerEngine.kt
index 6caff5e0..04d6d48d 100644
--- a/player/src/main/java/com/streamvault/player/PlayerEngine.kt
+++ b/player/src/main/java/com/streamvault/player/PlayerEngine.kt
@@ -10,6 +10,7 @@ import com.streamvault.domain.model.DrmScheme
import com.streamvault.domain.model.StreamInfo
import com.streamvault.domain.model.VideoFormat
import androidx.media3.common.PlaybackException
+import androidx.media3.common.text.Cue
import androidx.media3.datasource.HttpDataSource
import com.streamvault.player.playback.PlaybackErrorCategory
import com.streamvault.player.playback.PlayerErrorClassifier
@@ -94,6 +95,10 @@ interface PlayerEngine {
fun selectVideoTrack(trackId: String)
fun selectSubtitleTrack(trackId: String?) // null to disable subtitles
fun addExternalSubtitle(subtitleUri: android.net.Uri, language: String)
+ fun setInjectedSubtitleCues(cues: List<Cue>)
+ fun clearInjectedSubtitleCues()
+ fun setLiveAudioTap(tap: LiveAudioTap?)
+ fun clearLiveAudioTap() = setLiveAudioTap(null)
fun release()
fun resetForReuse() = release()
diff --git a/player/src/main/java/com/streamvault/player/playback/LiveAudioTapAudioSink.kt b/player/src/main/java/com/streamvault/player/playback/LiveAudioTapAudioSink.kt
new file mode 100644
index 00000000..97c113ba
--- /dev/null
+++ b/player/src/main/java/com/streamvault/player/playback/LiveAudioTapAudioSink.kt
@@ -0,0 +1,73 @@
+package com.streamvault.player.playback
+
+import androidx.media3.common.C
+import androidx.media3.common.Format
+import androidx.media3.common.util.UnstableApi
+import androidx.media3.exoplayer.audio.AudioSink
+import com.streamvault.player.LiveAudioPcmBuffer
+import com.streamvault.player.LiveAudioTap
+import java.nio.ByteBuffer
+
+@UnstableApi
+internal class LiveAudioTapAudioSink(
+ private val delegate: AudioSink,
+ private val tapProvider: () -> LiveAudioTap?
+) : AudioSink by delegate {
+ private var sampleRate: Int = Format.NO_VALUE
+ private var channelCount: Int = Format.NO_VALUE
+ private var encoding: Int = C.ENCODING_INVALID
+
+ override fun configure(inputFormat: Format, specifiedBufferSize: Int, outputChannels: IntArray?) {
+ sampleRate = inputFormat.sampleRate
+ channelCount = inputFormat.channelCount
+ encoding = inputFormat.pcmEncoding
+ delegate.configure(inputFormat, specifiedBufferSize, outputChannels)
+ }
+
+ override fun handleBuffer(
+ buffer: ByteBuffer,
+ presentationTimeUs: Long,
+ encodedAccessUnitCount: Int
+ ): Boolean {
+ val startPosition = buffer.position()
+ val copySource = buffer.asReadOnlyBuffer()
+ val handled = delegate.handleBuffer(buffer, presentationTimeUs, encodedAccessUnitCount)
+ copyConsumedForTap(copySource, startPosition, buffer.position(), presentationTimeUs)
+ return handled
+ }
+
+ private fun copyConsumedForTap(
+ buffer: ByteBuffer,
+ startPosition: Int,
+ endPosition: Int,
+ presentationTimeUs: Long
+ ) {
+ val tap = tapProvider() ?: return
+ if (encoding != C.ENCODING_PCM_16BIT || sampleRate <= 0 || channelCount <= 0 || endPosition <= startPosition) {
+ return
+ }
+ val adjustedPresentationTimeUs = presentationTimeUs + consumedOffsetUs(startPosition)
+ val duplicate = buffer.apply {
+ position(startPosition)
+ limit(endPosition)
+ }
+ val data = ByteArray(duplicate.remaining())
+ duplicate.get(data)
+ tap.onPcmAudio(
+ LiveAudioPcmBuffer(
+ data = data,
+ presentationTimeUs = adjustedPresentationTimeUs,
+ sampleRate = sampleRate,
+ channelCount = channelCount,
+ encoding = encoding
+ )
+ )
+ }
+
+ private fun consumedOffsetUs(bytePosition: Int): Long {
+ val frameSize = channelCount * 2
+ if (frameSize <= 0 || sampleRate <= 0) return 0L
+ val frames = bytePosition / frameSize
+ return frames * 1_000_000L / sampleRate
+ }
+}
diff --git a/player/src/main/java/com/streamvault/player/ui/PlayerViewBinder.kt b/player/src/main/java/com/streamvault/player/ui/PlayerViewBinder.kt
index 2d43ffc4..abf6360d 100644
--- a/player/src/main/java/com/streamvault/player/ui/PlayerViewBinder.kt
+++ b/player/src/main/java/com/streamvault/player/ui/PlayerViewBinder.kt
@@ -5,6 +5,7 @@ import android.graphics.Color
import android.view.LayoutInflater
import android.view.View
import android.widget.FrameLayout
+import androidx.media3.common.text.Cue
import androidx.media3.ui.AspectRatioFrameLayout
import androidx.media3.ui.PlayerView
import com.streamvault.player.PlayerRenderSurfaceType
@@ -16,6 +17,7 @@ class PlayerViewBinder(
) {
private var boundPlayerView: PlayerView? = null
private var boundResizeMode: PlayerSurfaceResizeMode = PlayerSurfaceResizeMode.FIT
+ private var injectedSubtitleCues: List<Cue> = emptyList()
fun createRenderView(
context: Context,
@@ -43,12 +45,14 @@ class PlayerViewBinder(
}
playerView.applyResizeMode(resizeMode)
subtitleStyleController.apply(playerView)
+ applyInjectedCues(playerView)
}
fun attachPlayer(player: androidx.media3.exoplayer.ExoPlayer?) {
boundPlayerView?.player = player
boundPlayerView?.applyResizeMode(boundResizeMode)
subtitleStyleController.apply(boundPlayerView)
+ applyInjectedCues(boundPlayerView)
}
fun release(renderView: View) {
@@ -67,6 +71,21 @@ class PlayerViewBinder(
fun reapplyStyle() {
subtitleStyleController.apply(boundPlayerView)
+ applyInjectedCues(boundPlayerView)
+ }
+
+ fun setInjectedSubtitleCues(cues: List<Cue>) {
+ injectedSubtitleCues = cues
+ applyInjectedCues(boundPlayerView)
+ }
+
+ fun clearInjectedSubtitleCues() {
+ injectedSubtitleCues = emptyList()
+ applyInjectedCues(boundPlayerView)
+ }
+
+ private fun applyInjectedCues(playerView: PlayerView?) {
+ playerView?.subtitleView?.setCues(injectedSubtitleCues)
}
private fun PlayerView.applyResizeMode(surfaceResizeMode: PlayerSurfaceResizeMode) {
diff --git a/tools/live-translation-service/.gitignore b/tools/live-translation-service/.gitignore
new file mode 100644
index 00000000..bd7a8363
--- /dev/null
+++ b/tools/live-translation-service/.gitignore
@@ -0,0 +1,4 @@
+.hf-cache/
+.venv/
+__pycache__/
+*.pyc
diff --git a/tools/live-translation-service/requirements.txt b/tools/live-translation-service/requirements.txt
new file mode 100644
index 00000000..4b31f46b
--- /dev/null
+++ b/tools/live-translation-service/requirements.txt
@@ -0,0 +1,15 @@
+# Live translation service dependencies.
+# CPU backend (default): faster-whisper / CTranslate2 (CPU-only on macOS).
+fastapi>=0.110
+uvicorn>=0.29
+numpy>=1.24
+faster-whisper>=1.2.1
+
+# Optional GPU (Metal) backend on Apple Silicon. Install to enable WHISPER_BACKEND=mlx:
+# pip install mlx-whisper
+# mlx-whisper>=0.4
+
+# Optional LLM translation backend (WHISPER_TRANSLATION_MODE=llm), talks to a local
+# OpenAI-compatible endpoint (LM Studio / Ollama):
+openai>=1.40
+httpx>=0.27
diff --git a/tools/live-translation-service/service.py b/tools/live-translation-service/service.py
new file mode 100644
index 00000000..e1fff690
--- /dev/null
+++ b/tools/live-translation-service/service.py
@@ -0,0 +1,778 @@
+import asyncio
+import dataclasses
+import logging
+import os
+import re
+import time
+import uuid
+from collections import deque
+from pathlib import Path
+from typing import Optional
+
+from fastapi import FastAPI, HTTPException, Query, Request
+from pydantic import BaseModel
+from faster_whisper import WhisperModel
+
+
+logger = logging.getLogger("live-translation")
+logging.basicConfig(level=logging.INFO)
+
+TARGET_SAMPLE_RATE = 16000
+DEFAULT_MODEL = os.environ.get("WHISPER_MODEL", "large-v3")
+
+# --- Compute backend / resource configuration (tuned for Apple Silicon) ---
+# NOTE: faster-whisper (CTranslate2) is CPU-only on macOS; it cannot use the Mac
+# GPU/Neural Engine. To actually use the M3 GPU, set WHISPER_BACKEND=mlx (requires
+# `pip install mlx-whisper`), which runs Whisper on Metal.
+BACKEND = os.environ.get("WHISPER_BACKEND", "faster-whisper").lower()
+# Quantization for the CPU (CTranslate2) backend. int8 is the fastest on CPU.
+COMPUTE_TYPE = os.environ.get("WHISPER_COMPUTE_TYPE", "int8")
+# Parallel inference workers (each can serve a concurrent request).
+NUM_WORKERS = int(os.environ.get("WHISPER_NUM_WORKERS", "1"))
+# MLX (GPU) model repo, used only when WHISPER_BACKEND=mlx.
+MLX_MODEL = os.environ.get("WHISPER_MLX_MODEL", "mlx-community/whisper-large-v3-mlx")
+
+# --- Translation mode ---
+# "whisper": Whisper itself translates (task=translate).
+# "llm": Whisper transcribes the source language, then a local OpenAI-compatible
+# LLM translates each finalised phrase to the target language. The LLM is
+# told not to repeat prior translations, which avoids re-showing text.
+TRANSLATION_MODE = os.environ.get("WHISPER_TRANSLATION_MODE", "whisper").lower()
+LLM_BASE_URL = os.environ.get("LLM_BASE_URL", "http://localhost:1234/v1")
+LLM_API_KEY = os.environ.get("LLM_API_KEY", "lm-studio")
+LLM_MODEL = os.environ.get("LLM_MODEL", "qwen3.6-35b-a3b")
+LLM_TARGET_LANG = os.environ.get("LLM_TARGET_LANG", "English")
+# Append "/no_think" and strip <think>...</think> for reasoning models (Qwen3 etc.).
+LLM_NO_THINK = os.environ.get("LLM_NO_THINK", "true").lower() == "true"
+
+
+def _detect_cpu_threads() -> int:
+ """Use all performance cores by default (avoid the slower efficiency cores)."""
+ env = os.environ.get("WHISPER_CPU_THREADS")
+ if env and env.isdigit() and int(env) > 0:
+ return int(env)
+ try:
+ import subprocess
+
+ out = subprocess.run(
+ ["sysctl", "-n", "hw.perflevel0.logicalcpu"],
+ capture_output=True,
+ text=True,
+ timeout=2,
+ )
+ cores = int(out.stdout.strip())
+ if cores > 0:
+ return cores
+ except Exception:
+ pass
+ return os.cpu_count() or 4
+
+
+CPU_THREADS = _detect_cpu_threads()
+
+# --- Phrase / VAD streaming parameters (adapted from Vanyoo/realtime-subtitle) ---
+# RMS amplitude (on float32 audio normalised to [-1, 1]) below which audio is silence.
+SILENCE_RMS = float(os.environ.get("WHISPER_SILENCE_RMS", "0.01"))
+_LLM_MODE = TRANSLATION_MODE == "llm"
+# Tail of silence (seconds) that, past a minimum phrase length, finalises a phrase.
+SILENCE_DURATION = float(os.environ.get("WHISPER_SILENCE_DURATION", "1.0" if _LLM_MODE else "0.8"))
+# Past this length, a short pause is enough to finalise (avoids huge latency).
+SOFT_LIMIT_SECONDS = float(os.environ.get("WHISPER_SOFT_LIMIT_SECONDS", "5" if _LLM_MODE else "6"))
+# Hard cap: force-finalise a phrase even with no pause.
+MAX_PHRASE_SECONDS = float(os.environ.get("WHISPER_MAX_PHRASE_SECONDS", "8" if _LLM_MODE else "12"))
+# Audio tail kept across phrase boundaries so words at cuts aren't lost.
+PHRASE_OVERLAP_SECONDS = float(os.environ.get("WHISPER_PHRASE_OVERLAP_SECONDS", "0.4"))
+# Minimum buffered audio before we emit a partial / finalise anything.
+MIN_PHRASE_SECONDS = 0.5
+# Minimum spacing (seconds) between partial (in-progress) transcriptions.
+UPDATE_INTERVAL = float(os.environ.get("WHISPER_UPDATE_INTERVAL", "1.2" if _LLM_MODE else "0.8"))
+# Only pin the source language once detection is confident, so a music/intro
+# window can't lock us onto the wrong language for the whole session.
+LANGUAGE_PIN_MIN_PROB = float(os.environ.get("WHISPER_LANGUAGE_PIN_MIN_PROB", "0.7"))
+
+
+class StartSessionRequest(BaseModel):
+ logicalStreamUrl: Optional[str] = None
+ providerId: Optional[int] = None
+ contentId: Optional[int] = None
+ targetLanguage: str = "en"
+ task: str = "translate"
+ model: Optional[str] = None
+ autoDetectSourceLanguage: bool = True
+
+
+class StartSessionResponse(BaseModel):
+ sessionId: str
+ status: str = "started"
+ model: str
+
+
+class TranslationUpdateDto(BaseModel):
+ sessionId: str
+ chunkId: int
+ isFinal: bool
+ text: str
+ sourceLanguage: Optional[str] = None
+
+
+class TranslationUpdate:
+ chunk_id: int
+ is_final: bool
+ text: str
+ source_language: Optional[str]
+
+
+class PendingWork:
+ snapshot: object
+ chunk_id: int
+ is_final: bool
+ buffer_duration: float
+ prompt: str
+ pinned: Optional[str]
+ use_llm: bool
+ asr_task: str
+ model_name: str
+
+
+class SessionState:
+ id: str
+ created_at_monotonic: float
+ model_name: str
+ task: str
+ lock: asyncio.Lock
+ closed: bool
+ # Accumulating PCM16 mono 16 kHz audio for the current phrase.
+ audio: bytearray
+ # Monotonically increasing id; one per finalised phrase.
+ chunk_id: int
+ # Last time a partial transcription was emitted (monotonic seconds).
+ last_partial_at: float
+ # Last finalised translation, reused as Whisper initial_prompt for continuity.
+ last_final_text: str
+ source_language: Optional[str]
+ # LLM-mode context: previous source text and its translation (for continuity
+ # and to instruct the LLM not to repeat).
+ prev_source: str
+ prev_translation: str
+ # Latest caption produced by the background worker; returned on the next upload
+ # once generation advances (so HTTP uploads stay fast and non-blocking).
+ latest_update: Optional[TranslationUpdate] = None
+ update_generation: int = 0
+ last_returned_generation: int = 0
+ # Finals are queued and never dropped; partials coalesce to the latest snapshot.
+ pending_queue: deque = dataclasses.field(default_factory=deque)
+ pending_partial: Optional[PendingWork] = None
+ process_task: Optional[asyncio.Task] = None
+
+
+def is_hallucination(text: str) -> bool:
+ """Detect Whisper repetition-loop hallucinations (e.g. 'once once once')."""
+ if not text:
+ return False
+ words = text.split()
+ if not words:
+ return False
+ max_repeats = 0
+ current = 1
+ last = ""
+ for word in words:
+ if word == last:
+ current += 1
+ else:
+ max_repeats = max(max_repeats, current)
+ current = 1
+ last = word
+ max_repeats = max(max_repeats, current)
+ if max_repeats > 4:
+ return True
+ if len(words) > 10:
+ ratio = len(set(words)) / len(words)
+ if ratio < 0.4:
+ return True
+ return False
+
+
+def is_prompt_echo(text: str, prompt: str) -> bool:
+ """Detect output that merely echoes the prompt (common on music/silence)."""
+ if not text or not prompt:
+ return False
+
+ def normalize(value: str) -> str:
+ return re.sub(r"[^\w\s]", "", value.lower()).strip()
+
+ norm_text = normalize(text)
+ norm_prompt = normalize(prompt)
+ if not norm_text or not norm_prompt:
+ return False
+ if norm_text == norm_prompt:
+ return True
+ if norm_prompt.endswith(norm_text):
+ return True
+ return False
+
+
+def _strip_thinking(text: str) -> str:
+ """Remove <think>...</think> blocks emitted by reasoning models."""
+ cleaned = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL)
+ # Drop any dangling unclosed think prefix.
+ cleaned = re.sub(r"^.*?</think>", "", cleaned, flags=re.DOTALL)
+ return cleaned.strip()
+
+
+class LlmTranslator:
+ """Translates text via a local OpenAI-compatible endpoint (e.g. LM Studio)."""
+
+ def __init__(self, base_url: str, api_key: str, model: str, target_lang: str) -> None:
+ from openai import OpenAI
+ import httpx
+
+ self.model = model
+ self.target_lang = target_lang
+ self.client = OpenAI(
+ api_key=api_key or "dummy",
+ base_url=base_url,
+ http_client=httpx.Client(verify=False),
+ )
+
+ def translate(
+ self,
+ text: str,
+ prev_source: str = "",
+ prev_translation: str = "",
+ *,
+ use_context: bool = True,
+ ) -> str:
+ if not text or not text.strip():
+ return ""
+ if use_context and prev_source and prev_translation:
+ system_prompt = (
+ f"You are a professional real-time translator. Translate the user "
+ f"message into {self.target_lang}.\n"
+ f"<context>\n"
+ f"Previous source: \"{prev_source}\"\n"
+ f"Previous translation: \"{prev_translation}\"\n"
+ f"</context>\n"
+ f"Rules:\n"
+ f"1. Use the context ONLY for terminology consistency.\n"
+ f"2. Translate ONLY the user message.\n"
+ f"3. Do NOT repeat the previous source or translation.\n"
+ f"4. Output ONLY the translation, with no notes or quotes."
+ )
+ else:
+ system_prompt = (
+ f"You are a professional real-time translator. Translate the user "
+ f"message into {self.target_lang}. Output ONLY the translation, with "
+ f"no notes, explanations, or quotes."
+ )
+ user_content = f"{text} /no_think" if LLM_NO_THINK else text
+ response = self.client.chat.completions.create(
+ model=self.model,
+ messages=[
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": user_content},
+ ],
+ temperature=0.2,
+ max_tokens=768,
+ timeout=30.0,
+ )
+ message = response.choices[0].message
+ raw = (message.content or "").strip()
+ if not raw:
+ reasoning = getattr(message, "reasoning_content", None) or ""
+ if reasoning.strip():
+ logger.warning("LLM returned empty content (reasoning only); retrying without context")
+ if use_context and (prev_source or prev_translation):
+ return self.translate(text, use_context=False)
+ return _strip_thinking(raw)
+
+
+class ModelHolder:
+ def __init__(self) -> None:
+ self._lock = asyncio.Lock()
+ self._model: Optional[WhisperModel] = None
+ self._model_name: Optional[str] = None
+
+ async def get(self, model_name: str) -> WhisperModel:
+ async with self._lock:
+ resolved_model = resolve_model_path(model_name)
+ if self._model is None or self._model_name != resolved_model:
+ logger.info(
+ "loading faster-whisper model=%s compute=%s cpu_threads=%d num_workers=%d",
+ resolved_model,
+ COMPUTE_TYPE,
+ CPU_THREADS,
+ NUM_WORKERS,
+ )
+ self._model = WhisperModel(
+ resolved_model,
+ device="cpu",
+ compute_type=COMPUTE_TYPE,
+ cpu_threads=CPU_THREADS,
+ num_workers=NUM_WORKERS,
+ local_files_only=Path(resolved_model).exists(),
+ )
+ self._model_name = resolved_model
+ return self._model
+
+
+def resolve_model_path(model_name: str) -> str:
+ if model_name not in {"large-v3", "whisper-large-v3"}:
+ return model_name
+ hf_home = Path(os.environ.get("HF_HOME", Path.home() / ".cache" / "huggingface"))
+ snapshot_root = hf_home / "hub" / "models--Systran--faster-whisper-large-v3" / "snapshots"
+ if snapshot_root.exists():
+ snapshots = sorted(snapshot_root.iterdir(), key=lambda path: path.stat().st_mtime, reverse=True)
+ if snapshots:
+ return str(snapshots[0])
+ return "large-v3"
+
+
+class SessionRegistry:
+ def __init__(self, model_holder: ModelHolder) -> None:
+ self._sessions: dict[str, SessionState] = {}
+ self._registry_lock = asyncio.Lock()
+ self._model_holder = model_holder
+
+ async def create(self, req: StartSessionRequest) -> SessionState:
+ session_id = str(uuid.uuid4())
+ model_name = req.model or DEFAULT_MODEL
+ state = SessionState(
+ id=session_id,
+ created_at_monotonic=time.monotonic(),
+ model_name=model_name,
+ task=req.task,
+ lock=asyncio.Lock(),
+ closed=False,
+ audio=bytearray(),
+ chunk_id=1,
+ last_partial_at=time.monotonic(),
+ last_final_text="",
+ source_language=None,
+ prev_source="",
+ prev_translation="",
+ )
+ async with self._registry_lock:
+ self._sessions[session_id] = state
+ return state
+
+ async def get(self, session_id: str) -> SessionState:
+ async with self._registry_lock:
+ state = self._sessions.get(session_id)
+ if state is None:
+ raise HTTPException(status_code=404, detail="session not found")
+ return state
+
+ async def delete(self, session_id: str) -> bool:
+ async with self._registry_lock:
+ state = self._sessions.pop(session_id, None)
+ if state is None:
+ return False
+ state.closed = True
+ task = state.process_task
+ if task is not None and not task.done():
+ task.cancel()
+ return True
+
+ async def _take_unreturned_update(self, state: SessionState) -> Optional[TranslationUpdate]:
+ async with state.lock:
+ if state.latest_update and state.update_generation > state.last_returned_generation:
+ state.last_returned_generation = state.update_generation
+ return state.latest_update
+ return None
+
+ def _enqueue_work(self, state: SessionState, work: PendingWork) -> None:
+ if work.is_final:
+ # A final supersedes any coalesced partial for the same phrase.
+ state.pending_partial = None
+ state.pending_queue.append(work)
+ logger.info(
+ "queued final chunk=%d dur=%.1fs (queue=%d)",
+ work.chunk_id,
+ work.buffer_duration,
+ len(state.pending_queue),
+ )
+ else:
+ state.pending_partial = work
+ self._schedule_processing(state)
+
+ def _schedule_processing(self, state: SessionState) -> None:
+ if state.process_task is not None and not state.process_task.done():
+ return
+ state.process_task = asyncio.create_task(self._process_pending(state))
+
+ async def _process_pending(self, state: SessionState) -> None:
+ try:
+ while True:
+ async with state.lock:
+ if state.closed:
+ state.process_task = None
+ return
+ work = None
+ if state.pending_queue:
+ work = state.pending_queue.popleft()
+ elif state.pending_partial is not None:
+ work = state.pending_partial
+ state.pending_partial = None
+ if work is None:
+ state.process_task = None
+ return
+
+ update = await self._run_work(state, work)
+ if update is not None:
+ async with state.lock:
+ state.latest_update = update
+ state.update_generation += 1
+ except asyncio.CancelledError:
+ async with state.lock:
+ state.process_task = None
+ raise
+ except Exception:
+ logger.exception("background translation failed for session=%s", state.id)
+ async with state.lock:
+ state.process_task = None
+
+ async def _run_work(self, state: SessionState, work: PendingWork) -> Optional[TranslationUpdate]:
+ # Phrase-level VAD already gates audio; inner Whisper VAD can drop words at
+ # boundaries when we manage our own phrase windows (especially in LLM mode).
+ skip_inner_vad = work.use_llm
+ if BACKEND == "mlx":
+ text, language, language_prob = await asyncio.to_thread(
+ self._transcribe_mlx,
+ work.snapshot,
+ work.asr_task,
+ work.prompt,
+ work.pinned,
+ skip_inner_vad,
+ )
+ else:
+ model = await self._model_holder.get(work.model_name)
+ text, language, language_prob = await asyncio.to_thread(
+ self._transcribe,
+ model,
+ work.snapshot,
+ work.asr_task,
+ work.prompt,
+ work.pinned,
+ skip_inner_vad,
+ )
+ logger.info(
+ "chunk=%d final=%s dur=%.1fs lang=%s p=%.2f text=%r",
+ work.chunk_id,
+ work.is_final,
+ work.buffer_duration,
+ language,
+ language_prob,
+ text,
+ )
+ if not text:
+ return None
+
+ async with state.lock:
+ if (
+ state.source_language is None
+ and language
+ and language_prob >= LANGUAGE_PIN_MIN_PROB
+ ):
+ state.source_language = language
+ prev_source = state.prev_source
+ prev_translation = state.prev_translation
+
+ if work.use_llm and translator is not None:
+ source_text = text
+ if work.is_final:
+ translated = await asyncio.to_thread(
+ translator.translate,
+ source_text,
+ prev_source,
+ prev_translation,
+ use_context=True,
+ )
+ if not translated.strip():
+ translated = await asyncio.to_thread(
+ translator.translate, source_text, use_context=False
+ )
+ translated = translated.strip()
+ if translated:
+ async with state.lock:
+ state.prev_source = source_text
+ state.prev_translation = translated
+ else:
+ translated = await asyncio.to_thread(
+ translator.translate, source_text, use_context=False
+ )
+ translated = translated.strip()
+ logger.info(
+ "chunk=%d llm final=%s: %r -> %r",
+ work.chunk_id,
+ work.is_final,
+ source_text,
+ translated,
+ )
+ if not translated:
+ return None
+ text = translated
+
+ return TranslationUpdate(
+ chunk_id=work.chunk_id,
+ is_final=work.is_final,
+ text=text,
+ source_language=language,
+ )
+
+ async def ingest_pcm_chunk(
+ self,
+ state: SessionState,
+ pcm_chunk: bytes,
+ start_ms: int,
+ ) -> Optional[TranslationUpdate]:
+ if state.closed:
+ raise HTTPException(status_code=410, detail="session closed")
+ if not pcm_chunk:
+ return await self._take_unreturned_update(state)
+
+ use_llm = TRANSLATION_MODE == "llm" and translator is not None
+
+ import numpy as np
+
+ async with state.lock:
+ state.audio.extend(pcm_chunk)
+ buf = np.frombuffer(bytes(state.audio), dtype=np.int16).astype(np.float32) / 32768.0
+ sample_rate = TARGET_SAMPLE_RATE
+ buffer_duration = len(buf) / sample_rate
+ now = time.monotonic()
+
+ def tail_rms(seconds: float) -> float:
+ samples = int(sample_rate * seconds)
+ if samples <= 0 or len(buf) < samples:
+ return 1.0
+ return float(np.sqrt(np.mean(buf[-samples:] ** 2)))
+
+ overall_rms = float(np.sqrt(np.mean(buf ** 2))) if len(buf) else 0.0
+
+ standard_cut = buffer_duration > 2.0 and tail_rms(SILENCE_DURATION) < SILENCE_RMS
+ soft_limit_cut = buffer_duration > SOFT_LIMIT_SECONDS and tail_rms(0.4) < SILENCE_RMS
+ hard_limit_cut = buffer_duration > MAX_PHRASE_SECONDS
+ should_finalize = standard_cut or soft_limit_cut or hard_limit_cut
+
+ should_queue = False
+ snapshot = None
+ chunk_id = state.chunk_id
+ prompt = ""
+ pinned = state.source_language
+ is_final = False
+
+ if should_finalize and buffer_duration > MIN_PHRASE_SECONDS:
+ snapshot = buf.copy()
+ chunk_id = state.chunk_id
+ overlap_bytes = int(TARGET_SAMPLE_RATE * PHRASE_OVERLAP_SECONDS) * 2
+ if overlap_bytes > 0 and len(state.audio) >= overlap_bytes:
+ state.audio = bytearray(state.audio[-overlap_bytes:])
+ else:
+ state.audio = bytearray()
+ state.chunk_id += 1
+ state.last_partial_at = now
+ is_final = True
+ should_queue = overall_rms >= SILENCE_RMS
+ elif (now - state.last_partial_at) >= UPDATE_INTERVAL and buffer_duration > MIN_PHRASE_SECONDS:
+ snapshot = buf.copy()
+ state.last_partial_at = now
+ is_final = False
+ should_queue = overall_rms >= SILENCE_RMS
+
+ if should_queue and snapshot is not None:
+ asr_task = "transcribe" if use_llm else state.task
+ self._enqueue_work(
+ state,
+ PendingWork(
+ snapshot=snapshot,
+ chunk_id=chunk_id,
+ is_final=is_final,
+ buffer_duration=buffer_duration,
+ prompt=prompt,
+ pinned=pinned,
+ use_llm=use_llm,
+ asr_task=asr_task,
+ model_name=state.model_name,
+ ),
+ )
+
+ return await self._take_unreturned_update(state)
+
+ @staticmethod
+ def _transcribe(
+ model: WhisperModel,
+ audio_f32,
+ task: str,
+ prompt: str,
+ language: Optional[str],
+ skip_inner_vad: bool = False,
+ ) -> tuple[str, Optional[str], float]:
+ segments, info = model.transcribe(
+ audio=audio_f32,
+ task="translate" if task == "translate" else "transcribe",
+ language=language,
+ beam_size=5,
+ temperature=0.0,
+ condition_on_previous_text=False,
+ initial_prompt=prompt or None,
+ no_speech_threshold=0.6,
+ log_prob_threshold=-1.0,
+ compression_ratio_threshold=2.4,
+ vad_filter=not skip_inner_vad,
+ vad_parameters=dict(threshold=0.5, min_silence_duration_ms=300),
+ )
+ detected_language = language or getattr(info, "language", None)
+ language_prob = float(getattr(info, "language_probability", 0.0) or 0.0)
+ text = " ".join((seg.text or "").strip() for seg in segments if (seg.text or "").strip()).strip()
+ if is_hallucination(text):
+ logger.info("filtered hallucination: %r", text[:60])
+ return "", detected_language, language_prob
+ if prompt and is_prompt_echo(text, prompt):
+ logger.info("filtered prompt echo: %r", text[:60])
+ return "", detected_language, language_prob
+ return text, detected_language, language_prob
+
+ @staticmethod
+ def _transcribe_mlx(
+ audio_f32,
+ task: str,
+ prompt: str,
+ language: Optional[str],
+ skip_inner_vad: bool = False,
+ ) -> tuple[str, Optional[str], float]:
+ import mlx_whisper
+
+ _ = skip_inner_vad # mlx-whisper has no VAD filter; phrase VAD is external.
+ result = mlx_whisper.transcribe(
+ audio_f32,
+ path_or_hf_repo=MLX_MODEL,
+ task="translate" if task == "translate" else "transcribe",
+ language=language,
+ temperature=0.0,
+ condition_on_previous_text=False,
+ initial_prompt=prompt or None,
+ no_speech_threshold=0.6,
+ )
+ text = (result.get("text") or "").strip()
+ detected_language = language or result.get("language")
+ # MLX doesn't return a language probability; treat a detected language as
+ # confident enough to pin (the phrase/silence checks already gate this).
+ language_prob = 1.0 if detected_language else 0.0
+ if is_hallucination(text):
+ logger.info("filtered hallucination: %r", text[:60])
+ return "", detected_language, language_prob
+ if prompt and is_prompt_echo(text, prompt):
+ logger.info("filtered prompt echo: %r", text[:60])
+ return "", detected_language, language_prob
+ return text, detected_language, language_prob
+
+
+app = FastAPI(title="Live Translation Service")
+model_holder = ModelHolder()
+
+
+def _build_translator() -> Optional[LlmTranslator]:
+ if TRANSLATION_MODE != "llm":
+ return None
+ try:
+ translator = LlmTranslator(LLM_BASE_URL, LLM_API_KEY, LLM_MODEL, LLM_TARGET_LANG)
+ logger.info("LLM translation enabled: model=%s url=%s", LLM_MODEL, LLM_BASE_URL)
+ return translator
+ except Exception as exc: # noqa: BLE001
+ logger.error("failed to init LLM translator (%s); falling back to whisper translate", exc)
+ return None
+
+
+translator = _build_translator()
+registry = SessionRegistry(model_holder)
+
+
+def health() -> dict:
+ return {
+ "status": "ok",
+ "model": DEFAULT_MODEL,
+ "backend": BACKEND,
+ "cpuThreads": CPU_THREADS,
+ "numWorkers": NUM_WORKERS,
+ "computeType": COMPUTE_TYPE,
+ "translationMode": TRANSLATION_MODE,
+ "llmModel": LLM_MODEL if TRANSLATION_MODE == "llm" else None,
+ "llmReady": translator is not None,
+ }
+
+
[email protected]_event("startup")
+async def _warmup() -> None:
+ """Preload + warm the model so the first real phrase isn't slow."""
+ import numpy as np
+
+ silence = np.zeros(TARGET_SAMPLE_RATE, dtype=np.float32)
+ try:
+ if BACKEND == "mlx":
+ await asyncio.to_thread(
+ SessionRegistry._transcribe_mlx, silence, "translate", "", None
+ )
+ logger.info("MLX (GPU) backend warmed: model=%s", MLX_MODEL)
+ else:
+ model = await model_holder.get(DEFAULT_MODEL)
+ await asyncio.to_thread(
+ SessionRegistry._transcribe, model, silence, "translate", "", None
+ )
+ logger.info(
+ "faster-whisper (CPU) backend warmed: cpu_threads=%d", CPU_THREADS
+ )
+ except Exception as exc: # noqa: BLE001 - warmup is best-effort
+ logger.warning("warmup skipped: %s", exc)
+
+
[email protected]("/v1/live-translation/session", response_model=StartSessionResponse)
+async def start_session(req: StartSessionRequest) -> StartSessionResponse:
+ model_name = req.model or DEFAULT_MODEL
+ if model_name == "whisper-large-v3":
+ model_name = "large-v3"
+ elif model_name not in {"large-v3"}:
+ model_name = "large-v3"
+ state = await registry.create(req.model_copy(update={"model": model_name}))
+ return StartSessionResponse(sessionId=state.id, model=model_name)
+
+
[email protected]("/v1/live-translation/session/{session_id}/audio", response_model=TranslationUpdateDto)
+async def upload_audio(
+ session_id: str,
+ request: Request,
+ start_ms: int = Query(alias="startMs"),
+ end_ms: int = Query(alias="endMs"),
+ sample_rate: int = Query(alias="sampleRate"),
+ channel_count: int = Query(alias="channelCount"),
+ encoding: str = Query(default="pcm_s16le"),
+) -> TranslationUpdateDto:
+ if sample_rate != TARGET_SAMPLE_RATE or channel_count != 1 or encoding != "pcm_s16le":
+ raise HTTPException(status_code=400, detail="expected 16 kHz mono pcm_s16le audio")
+ state = await registry.get(session_id)
+ pcm_chunk = await request.body()
+ update = await registry.ingest_pcm_chunk(state, pcm_chunk, start_ms)
+ if update is None:
+ return TranslationUpdateDto(
+ sessionId=session_id,
+ chunkId=state.chunk_id,
+ isFinal=False,
+ text="",
+ sourceLanguage=state.source_language,
+ )
+ return TranslationUpdateDto(
+ sessionId=session_id,
+ chunkId=update.chunk_id,
+ isFinal=update.is_final,
+ text=update.text,
+ sourceLanguage=update.source_language,
+ )
+
+
[email protected]("/v1/live-translation/session/{session_id}")
+async def delete_session(session_id: str) -> dict:
+ deleted = await registry.delete(session_id)
+ return {"deleted": deleted}