Unofficial YouTube Music / Spotify Premium client for Android, built in Liquid Glass. Audio effects, Spotify Connect, listening history, 8 languages. Rust core on librespot.
Kotlin alternativeandroidandroid-applicationclientglassmorphisminnertubelibrespotliquid-glassspotifyspotify-connectspotifyapiyoutube-musicyoutube-playerytdlnis
Watch the video a song has
Lelonio0ab909c1247a
diff --git a/app/build.gradle.kts b/app/build.gradle.ktsindex 9907c88..848fca6 100644--- a/app/build.gradle.kts+++ b/app/build.gradle.kts@@ -181,6 +181,9 @@ dependencies {implementation(libs.media3.common)implementation(libs.media3.session)implementation(libs.media3.exoplayer)+ // DASH, for Spotify's own video: its manifest is a set of segment+ // templates, which is what a DASH source is built to walk.+ implementation(libs.media3.exoplayer.dash)implementation(libs.media3.ui)implementation(libs.kotlinx.coroutines.android)diff --git a/app/src/main/java/dev/lelonio/square/backend/spotify/SpotifyVideo.kt b/app/src/main/java/dev/lelonio/square/backend/spotify/SpotifyVideo.ktnew file mode 100644index 0000000..46dc7a5--- /dev/null+++ b/app/src/main/java/dev/lelonio/square/backend/spotify/SpotifyVideo.kt@@ -0,0 +1,257 @@+package dev.lelonio.square.backend.spotify++import kotlinx.coroutines.Dispatchers+import kotlinx.coroutines.withContext+import kotlinx.serialization.SerialName+import kotlinx.serialization.Serializable+import kotlinx.serialization.json.Json+import okhttp3.OkHttpClient+import okhttp3.Request++/**+ * The video a Spotify track has, when it has one.+ *+ * A track that comes with a video carries the id of it in the metadata the+ * context player already receives — `media.manifest_id`. That id is all this+ * needs: one request returns everything about the video, and everything about+ * the video is templates rather than a list of files.+ *+ * The shape is Spotify's own, not DASH: a set of profiles (the quality ladder),+ * two CDN roots, and two URL templates with `{{profile_id}}`, `{{file_type}}`+ * and `{{segment_timestamp}}` in them. [toMpd] turns that into the DASH a+ * player understands; see the note there for why translating beats writing a+ * media source by hand.+ */+object SpotifyVideo {++ /** Where the manifest lives. The access point's own host answers it too. */+ private const val HOST = "https://spclient.wg.spotify.com"++ private val http = OkHttpClient()++ private val json = Json { ignoreUnknownKeys = true }++ /**+ * Reads the manifest for a video file id.+ *+ * The listener's ordinary access token is enough: the client token every+ * Spotify client also sends is not asked for here, which was worth+ * checking before building anything on top.+ */+ suspend fun manifest(fileId: String, accessToken: String): VideoManifest =+ withContext(Dispatchers.IO) {+ val request = Request.Builder()+ .url("$HOST/manifests/v9/json/sources/$fileId/options/supports_drm")+ .header("authorization", "Bearer $accessToken")+ .header("accept", "application/json")+ .build()++ http.newCall(request).execute().use { response ->+ val body = response.body?.string().orEmpty()+ check(response.isSuccessful) {+ "manifest ${response.code} for $fileId"+ }+ json.decodeFromString<VideoManifest>(body)+ }+ }+}++@Serializable+data class VideoManifest(+ val contents: List<VideoContent> = emptyList(),+ @SerialName("base_urls") val baseUrls: List<String> = emptyList(),+ @SerialName("initialization_template") val initTemplate: String = "",+ @SerialName("segment_template") val segmentTemplate: String = "",+ @SerialName("end_time_millis") val endTimeMillis: Long = 0,+ val seekpanels: SeekPanels? = null,+)++/**+ * The thumbnails behind the scrubber: sheets of small frames, in the clear.+ *+ * Worth more than a preview here. The video itself is decrypted into a+ * protected buffer that nothing can read back — which is what stops the glow+ * the video player draws for YouTube — while these are ordinary JPEGs on a+ * CDN, one frame a second along the whole song. Sampled, they give the same+ * effect from the same pictures, without ever touching the protected path.+ */+@Serializable+data class SeekPanels(+ @SerialName("base_urls") val baseUrls: List<String> = emptyList(),+ val templates: List<String> = emptyList(),+ val variants: List<SeekPanelVariant> = emptyList(),+)++@Serializable+data class SeekPanelVariant(+ val id: String = "0",+ @SerialName("frame_width") val frameWidth: Int = 0,+ @SerialName("frame_height") val frameHeight: Int = 0,+ val rows: Int = 0,+ val columns: Int = 0,+)++/**+ * Where the thumbnail for a moment lives, and where it sits in that sheet.+ *+ * One frame a second, laid out in sheets of `rows * columns`: the second gives+ * the frame, the frame gives the sheet and the square inside it.+ */+data class PanelFrame(val url: String, val left: Int, val top: Int, val width: Int, val height: Int)++fun VideoManifest.panelAt(positionMs: Long): PanelFrame? {+ val panels = seekpanels ?: return null+ val base = panels.baseUrls.firstOrNull() ?: return null+ val template = panels.templates.firstOrNull() ?: return null+ val variant = panels.variants.firstOrNull() ?: return null+ if (variant.rows <= 0 || variant.columns <= 0) return null++ val perSheet = variant.rows * variant.columns+ val frame = (positionMs / 1000L).toInt().coerceAtLeast(0)+ val sheet = frame / perSheet+ val within = frame % perSheet++ val url = base.trimEnd('/') + "/" + template+ .replace("{{variant_id}}", variant.id)+ .replace("{{panel_id}}", sheet.toString())++ return PanelFrame(+ url = url,+ left = (within % variant.columns) * variant.frameWidth,+ top = (within / variant.columns) * variant.frameHeight,+ width = variant.frameWidth,+ height = variant.frameHeight,+ )+}++@Serializable+data class VideoContent(+ @SerialName("encoding_id") val encodingId: String = "",+ /** Seconds of media in each segment, and the step the template counts by. */+ @SerialName("segment_length") val segmentLength: Int = 4,+ @SerialName("end_time_millis") val endTimeMillis: Long = 0,+ val profiles: List<VideoProfile> = emptyList(),+ @SerialName("encryption_infos") val encryption: List<VideoEncryption> = emptyList(),+)++@Serializable+data class VideoProfile(+ val id: Int = 0,+ @SerialName("file_type") val fileType: String = "",+ @SerialName("mime_type") val mimeType: String = "",+ @SerialName("video_codec") val videoCodec: String? = null,+ @SerialName("audio_codec") val audioCodec: String? = null,+ @SerialName("video_width") val width: Int = 0,+ @SerialName("video_height") val height: Int = 0,+ @SerialName("video_bitrate") val videoBitrate: Int = 0,+ @SerialName("audio_bitrate") val audioBitrate: Int = 0,+ @SerialName("max_bitrate") val maxBitrate: Int = 0,+)++@Serializable+data class VideoEncryption(+ @SerialName("key_system") val keySystem: String = "",+ @SerialName("encryption_scheme") val scheme: String = "",+ @SerialName("license_server_endpoint") val licenseEndpoint: String = "",+ /** The PSSH box, base64, exactly as a player wants to be handed it. */+ @SerialName("encryption_data") val encryptionData: String = "",+)++/** Widevine's own, which is what an Android device can play. */+val VideoManifest.widevine: VideoEncryption?+ get() = contents.firstOrNull()?.encryption?.firstOrNull { it.keySystem == "widevine" }++/** Where the licence for this video is asked for. */+fun VideoManifest.licenseUrl(): String? =+ widevine?.licenseEndpoint?.let { "https://spclient.wg.spotify.com$it" }++/**+ * The same video, written as DASH.+ *+ * Translating rather than writing a media source by hand, and the reason is+ * that this manifest already *is* DASH in everything but spelling: a duration,+ * fixed-length segments, a ladder of renditions, and one template per kind of+ * file. What DASH calls a SegmentTemplate with `$Time$` is what Spotify calls+ * `{{segment_timestamp}}`, counting in seconds; the rest is renaming.+ *+ * The mp4 profiles only. They carry the same content as the WebM ones and are+ * encrypted the way every DASH player already knows how to decrypt — common+ * encryption, with the key box handed over in the manifest below.+ */+fun VideoManifest.toMpd(): String {+ val content = contents.firstOrNull() ?: error("manifest with no content")+ val base = baseUrls.firstOrNull() ?: error("manifest with no CDN")+ val seconds = content.segmentLength.coerceAtLeast(1)+ val duration = (content.endTimeMillis.takeIf { it > 0 } ?: endTimeMillis) / 1000.0++ val mp4 = content.profiles.filter { it.fileType == "mp4" }+ val video = mp4.filter { it.mimeType.startsWith("video/") }.sortedBy { it.videoBitrate }+ val audio = mp4.filter { it.mimeType.startsWith("audio/") }+ check(video.isNotEmpty() && audio.isNotEmpty()) { "no mp4 ladder in this manifest" }++ // A representation is named by its profile id, which is exactly what the+ // template wants where it says `{{profile_id}}` — so the two line up+ // without a table to keep in step.+ fun template(source: String) = source+ .replace("{{profile_id}}", "\$RepresentationID\$")+ .replace("{{file_type}}", "mp4")+ .replace("{{segment_timestamp}}", "\$Time\$")+ .xml()++ val protection = widevine?.let { widevine ->+ """+ <ContentProtection schemeIdUri="urn:mpeg:dash:mp4protection:2011" value="cenc"/>+ <ContentProtection schemeIdUri="urn:uuid:$WIDEVINE_UUID">+ <cenc:pssh>${widevine.encryptionData}</cenc:pssh>+ </ContentProtection>+ """.trimIndent()+ }.orEmpty()++ val segments = """+ <SegmentTemplate timescale="1" duration="$seconds" startNumber="0"+ initialization="${template(initTemplate)}"+ media="${template(segmentTemplate)}"/>+ """.trimIndent()++ fun representation(profile: VideoProfile): String = if (profile.width > 0) {+ """<Representation id="${profile.id}" mimeType="video/mp4" codecs="${profile.videoCodec}" """ ++ """width="${profile.width}" height="${profile.height}" """ ++ """bandwidth="${profile.videoBitrate.takeIf { it > 0 } ?: profile.maxBitrate}"/>"""+ } else {+ """<Representation id="${profile.id}" mimeType="audio/mp4" codecs="${profile.audioCodec}" """ ++ """audioSamplingRate="44100" """ ++ """bandwidth="${profile.audioBitrate.takeIf { it > 0 } ?: profile.maxBitrate}">""" ++ """<AudioChannelConfiguration """ ++ """schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="2"/>""" ++ """</Representation>"""+ }++ return """+<?xml version="1.0" encoding="utf-8"?>+<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:cenc="urn:mpeg:cenc:2013"+ profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" type="static"+ mediaPresentationDuration="PT${"%.3f".format(java.util.Locale.ROOT, duration)}S" minBufferTime="PT4S">+ <Period>+ <BaseURL>${base.xml()}</BaseURL>+ <AdaptationSet contentType="video" segmentAlignment="true" startWithSAP="1">+ $protection+ $segments+ ${video.joinToString("\n ") { representation(it) }}+ </AdaptationSet>+ <AdaptationSet contentType="audio" segmentAlignment="true" startWithSAP="1" lang="und">+ $protection+ $segments+ ${audio.joinToString("\n ") { representation(it) }}+ </AdaptationSet>+ </Period>+</MPD>+""".trimIndent()+}++/** Widevine's system id, as DASH spells it. */+private const val WIDEVINE_UUID = "edef8ba9-79d6-4ace-a3c8-27dcd51d21ed"++/**+ * The signed URLs carry query strings, and a bare `&` ends an XML document.+ */+private fun String.xml(): String = replace("&", "&")diff --git a/app/src/main/java/dev/lelonio/square/backend/spotify/SpotifyVideoMode.kt b/app/src/main/java/dev/lelonio/square/backend/spotify/SpotifyVideoMode.ktnew file mode 100644index 0000000..35982f8--- /dev/null+++ b/app/src/main/java/dev/lelonio/square/backend/spotify/SpotifyVideoMode.kt@@ -0,0 +1,100 @@+package dev.lelonio.square.backend.spotify++import kotlinx.coroutines.flow.MutableSharedFlow+import kotlinx.coroutines.flow.MutableStateFlow+import kotlinx.coroutines.flow.SharedFlow+import kotlinx.coroutines.flow.StateFlow+import kotlinx.coroutines.flow.asSharedFlow+import kotlinx.coroutines.flow.asStateFlow++/**+ * Whether a Spotify track is being watched rather than listened to.+ *+ * Process-wide, like [dev.lelonio.square.backend.youtube.YouTubeVideoMode] and+ * for the same reason: the button is in the player and the thing that has to+ * act on it is the playback service, and no session command carries "play the+ * video of this instead".+ *+ * The video is not a second player running beside the music. It replaces what+ * the session is playing, carries the song's own audio, and gives the transport+ * controls something real to control — which is how the official client does it+ * too, and the only arrangement where pause means pause.+ *+ * Deliberately not persisted: video is something turned on for one song.+ */+object SpotifyVideoMode {++ /** What the player was asked to do; read by the service. */+ sealed interface Request {+ data class Watch(val fileId: String, val positionMs: Long) : Request+ data class Listen(val positionMs: Long) : Request++ /**+ * Leave the video and move along the queue.+ *+ * Skipping is about the song, not about the picture: the video is one+ * item and its own player has nowhere to go, while the queue the+ * listener is actually in is held by the engine parked behind it.+ */+ data class Skip(val forward: Boolean) : Request+ }++ private val _requests = MutableSharedFlow<Request>(extraBufferCapacity = 4)+ val requests: SharedFlow<Request> = _requests.asSharedFlow()++ private val _manifest = MutableStateFlow<VideoManifest?>(null)++ /**+ * The manifest of what is being watched.+ *+ * Kept because the picture is not the only thing in it: the thumbnail+ * sheets are what the glow is sampled from, the video's own pixels being+ * unreadable by design.+ */+ val manifest: StateFlow<VideoManifest?> = _manifest.asStateFlow()++ fun setManifest(manifest: VideoManifest?) {+ _manifest.value = manifest+ }++ private val _enabled = MutableStateFlow(false)++ /** True while the video is what the session is playing. */+ val enabled: StateFlow<Boolean> = _enabled.asStateFlow()++ fun watch(fileId: String, positionMs: Long) {+ _requests.tryEmit(Request.Watch(fileId, positionMs))+ }++ fun listen(positionMs: Long) {+ _requests.tryEmit(Request.Listen(positionMs))+ }++ fun skip(forward: Boolean) {+ _requests.tryEmit(Request.Skip(forward))+ }++ private val _generation = MutableStateFlow(0)++ /**+ * Bumped every time a video actually starts.+ *+ * The picture is drawn by attaching a surface to whatever the session is+ * playing, and that attachment is made once per player. Here the player is+ * swapped underneath — the engine cannot show video, so something else has+ * to be in front — and the surface would stay attached to the one that+ * left: sound, and a black rectangle. Screens watch this and attach again.+ */+ val generation: StateFlow<Int> = _generation.asStateFlow()++ /** Set by the service once the swap has actually happened. */+ fun setEnabled(enabled: Boolean) {+ _enabled.value = enabled+ if (enabled) _generation.value = _generation.value + 1+ }++ /** Back to sound alone, for when the track or the source changes under it. */+ fun reset() {+ _enabled.value = false+ }+}diff --git a/app/src/main/java/dev/lelonio/square/nativecore/NativeBridge.kt b/app/src/main/java/dev/lelonio/square/nativecore/NativeBridge.ktindex a0a23a9..b3f7e48 100644--- a/app/src/main/java/dev/lelonio/square/nativecore/NativeBridge.kt+++ b/app/src/main/java/dev/lelonio/square/nativecore/NativeBridge.kt@@ -318,6 +318,20 @@ object NativeBridge {*/fun playingHere(): String = runCatching { nativePlayingHere() }.getOrDefault("{}")+ /**+ * The music video for a track, as `{trackUri, fileId}`, or null.+ *+ * Null is the ordinary answer: almost no track has one.+ */+ /**+ * A token good for spclient, taken from the engine rather than the app's+ * own sign-in, which lapses while playback goes on working.+ */+ fun accessToken(): String? = runCatching { nativeAccessToken() }.getOrNull()++ fun trackVideo(uri: String): String? =+ runCatching { nativeTrackVideo(uri) }.getOrNull()?.takeIf { it != "null" }+/** Every device the account can see, this one included, as a JSON array. */fun remoteDevices(): String = nativeRemoteDevices()@@ -453,6 +467,12 @@ object NativeBridge {private external fun nativeDeviceId(): Stringprivate external fun nativeSetQueueOrder(urisJson: String, index: Int)private external fun nativePlayingHere(): String++ /** A token from the engine's own session; see catalog.rs. */+ private external fun nativeAccessToken(): String++ /** The music video a track has, as JSON, or "null"; see catalog.rs. */+ private external fun nativeTrackVideo(uri: String): Stringprivate external fun nativeRemoteState(): Stringprivate external fun nativeRemoteDevices(): Stringprivate external fun nativeRemoteCommand(deviceId: String, body: String)diff --git a/app/src/main/java/dev/lelonio/square/playback/LocalPlayerFactory.kt b/app/src/main/java/dev/lelonio/square/playback/LocalPlayerFactory.ktindex 89dd055..3419014 100644--- a/app/src/main/java/dev/lelonio/square/playback/LocalPlayerFactory.kt+++ b/app/src/main/java/dev/lelonio/square/playback/LocalPlayerFactory.kt@@ -64,7 +64,7 @@ object LocalPlayerFactory {* the same amount would sound worse from the phone than from Spotify. See* BungeeAudioProcessor.*/- private fun renderers(host: PlaybackHost) =+ internal fun renderers(host: PlaybackHost) =object : androidx.media3.exoplayer.DefaultRenderersFactory(host.context) {override fun buildAudioSink(context: android.content.Context,diff --git a/app/src/main/java/dev/lelonio/square/playback/PlaybackService.kt b/app/src/main/java/dev/lelonio/square/playback/PlaybackService.ktindex 3d7aa22..c6cfffd 100644--- a/app/src/main/java/dev/lelonio/square/playback/PlaybackService.kt+++ b/app/src/main/java/dev/lelonio/square/playback/PlaybackService.kt@@ -8,6 +8,8 @@ import dev.lelonio.square.auth.SpotifyOAuthimport dev.lelonio.square.auth.TokenStoreimport dev.lelonio.square.data.CatalogTrackimport dev.lelonio.square.data.PlaybackStore+import dev.lelonio.square.backend.spotify.licenseUrl+import dev.lelonio.square.backend.spotify.toMpdimport dev.lelonio.square.data.SavedPlaybackimport dev.lelonio.square.nativecore.NativeBridgeimport kotlinx.coroutines.CoroutineScope@@ -56,7 +58,16 @@ class PlaybackService : MediaLibraryService() {*/private var librespot: LibrespotPlayer? = null- private val queue: PlayQueue? get() = librespot?.let { container.spotifyBackend.queue }+ /**+ * The Spotify queue, including while something else is in front of it.+ *+ * A local file or a video is a player with one item in it, and saving+ * *that* as the queue is how the app came back from a video with the song+ * on screen and nothing behind it. The engine parked behind still holds the+ * real list; see [parkedSpotify].+ */+ private val queue: PlayQueue?+ get() = (librespot ?: parkedSpotify)?.let { container.spotifyBackend.queue }/** Owns the AudioTrack the native sink writes into; Spotify's alone. */private val audioOutput get() = container.spotifyBackend.audioOutput@@ -173,6 +184,32 @@ class PlaybackService : MediaLibraryService() {// network was a second of silence for an answer that is often wrong,// and the engine can now be told a new bitrate without one.+ // Watching, and going back to listening. The video replaces what the+ // session plays rather than running beside it, so this is a swap of the+ // player like changing source — see SpotifyVideoMode.+ scope.launch {+ dev.lelonio.square.backend.spotify.SpotifyVideoMode.requests.collect { request ->+ runCatching { playVideoRequest(request) }+ .onFailure { android.util.Log.w(TAG, "video: $it") }+ }+ }++ // Speed and pitch follow the sliders on whichever player is in front.+ //+ // The engine reads them from its own output, live. Everything else is+ // an ExoPlayer, which was given them once when it was built — so moving+ // a slider during a local file or a video did nothing at all until the+ // next track.+ scope.launch {+ kotlinx.coroutines.flow.combine(+ AudioEffects.speed,+ AudioEffects.pitch,+ ) { speed, pitch -> speed to pitch }.collect { (speed, pitch) ->+ (player as? androidx.media3.exoplayer.ExoPlayer)?.playbackParameters =+ androidx.media3.common.PlaybackParameters(speed, pitch)+ }+ }+// The room follows the slider on whichever player is in front. The// Spotify path takes it below, in its own output; this is for the rest.scope.launch {@@ -243,7 +280,7 @@ class PlaybackService : MediaLibraryService() {* neither backend's player can open them. So what decides this is not only* the setting but what is being played; see [ensurePlayerFor].*/- private enum class PlayerKind { SPOTIFY, YOUTUBE, LOCAL }+ private enum class PlayerKind { SPOTIFY, YOUTUBE, LOCAL, VIDEO }private var playerKind = PlayerKind.SPOTIFY@@ -266,6 +303,79 @@ class PlaybackService : MediaLibraryService() {*/private val exoReverb = PlayerReverb()++ /**+ * Swaps between the song and its video, keeping the position.+ *+ * The manifest and the licence both come from the engine's own token: the+ * app's sign-in lapses while playback goes on working, and a video that+ * stops working on those days would be a video nobody trusts.+ */+ private suspend fun playVideoRequest(+ request: dev.lelonio.square.backend.spotify.SpotifyVideoMode.Request,+ ) {+ when (request) {+ is dev.lelonio.square.backend.spotify.SpotifyVideoMode.Request.Watch -> {+ val token = withContext(Dispatchers.IO) { NativeBridge.accessToken() }+ ?: error("the engine has no token")+ val manifest = dev.lelonio.square.backend.spotify.SpotifyVideo+ .manifest(request.fileId, token)+ val file = withContext(Dispatchers.IO) {+ cacheDir.resolve("video-${request.fileId}.mpd")+ .apply { writeText(manifest.toMpd()) }+ }++ // The song's own name, artist and cover, carried over.+ //+ // Without them the video is a media item with nothing on it,+ // and everything that reads the session — this app's own+ // player, the notification, the lock screen, the car — shows a+ // blank where the song was. It is the same song; only the+ // source changed.+ val playing = player.currentMediaItem+ val identity = playing?.mediaId+ val metadata = playing?.mediaMetadata++ swapPlayer(PlayerKind.VIDEO, restore = false)+ val item = androidx.media3.common.MediaItem.Builder()+ .setUri(android.net.Uri.fromFile(file))+ .setMediaId(identity ?: request.fileId)+ .apply { metadata?.let(::setMediaMetadata) }+ .setMimeType(androidx.media3.common.MimeTypes.APPLICATION_MPD)+ .setDrmConfiguration(+ androidx.media3.common.MediaItem.DrmConfiguration+ .Builder(androidx.media3.common.C.WIDEVINE_UUID)+ .setLicenseUri(manifest.licenseUrl())+ .setLicenseRequestHeaders(mapOf("authorization" to "Bearer $token"))+ .build(),+ )+ .build()+ player.setMediaItem(item)+ player.prepare()+ player.seekTo(request.positionMs)+ player.play()+ dev.lelonio.square.backend.spotify.SpotifyVideoMode.setManifest(manifest)+ dev.lelonio.square.backend.spotify.SpotifyVideoMode.setEnabled(true)+ }++ is dev.lelonio.square.backend.spotify.SpotifyVideoMode.Request.Skip -> {+ dev.lelonio.square.backend.spotify.SpotifyVideoMode.setEnabled(false)+ dev.lelonio.square.backend.spotify.SpotifyVideoMode.setManifest(null)+ swapPlayer(kindFor(container.preferences.backend.value), restore = true)+ if (request.forward) player.seekToNext() else player.seekToPrevious()+ player.play()+ }++ is dev.lelonio.square.backend.spotify.SpotifyVideoMode.Request.Listen -> {+ dev.lelonio.square.backend.spotify.SpotifyVideoMode.setEnabled(false)+ dev.lelonio.square.backend.spotify.SpotifyVideoMode.setManifest(null)+ swapPlayer(kindFor(container.preferences.backend.value), restore = true)+ player.seekTo(request.positionMs)+ player.play()+ }+ }+ }+/** Puts the room on the player in front, or takes it off the one leaving. */private fun followReverb() {val exo = player as? androidx.media3.exoplayer.ExoPlayer ?: return@@ -325,6 +435,35 @@ class PlaybackService : MediaLibraryService() {}}+ // The video is an ordinary ExoPlayer: what makes it a video is the+ // media item it is given, which carries the DASH manifest and the+ // licence to decrypt it. See SpotifyVideo.+ PlayerKind.VIDEO -> {+ librespot = null+ playerKind = kind+ // The same sink as everything else that is not the engine, which is+ // where the app's own speed and pitch live: built plain, a video+ // played at one and dry while the listener had the sliders set, and+ // the effects came back only when the song did.+ androidx.media3.exoplayer.ExoPlayer.Builder(this, LocalPlayerFactory.renderers(playbackHost))+ .setLooper(playbackHost.looper)+ .setHandleAudioBecomingNoisy(true)+ .setAudioAttributes(+ androidx.media3.common.AudioAttributes.Builder()+ .setUsage(androidx.media3.common.C.USAGE_MEDIA)+ .setContentType(androidx.media3.common.C.AUDIO_CONTENT_TYPE_MUSIC)+ .build(),+ /* handleAudioFocus = */ true,+ )+ .build()+ .also { built ->+ built.playbackParameters = androidx.media3.common.PlaybackParameters(+ AudioEffects.speed.value,+ AudioEffects.pitch.value,+ )+ }+ }+PlayerKind.SPOTIFY -> buildBackendPlayer(dev.lelonio.square.backend.BackendId.SPOTIFY)PlayerKind.YOUTUBE -> buildBackendPlayer(dev.lelonio.square.backend.BackendId.YOUTUBE_MUSIC)}@@ -417,18 +556,18 @@ class PlaybackService : MediaLibraryService() {// already configured.(player as? androidx.media3.exoplayer.ExoPlayer)?.let(exoReverb::release)- val parking = librespot != null && kind == PlayerKind.LOCAL+ // The video is the same case as a local file: the session plays+ // something else for a while, and the engine has to be there when the+ // song comes back — it is also what the whole catalogue is read+ // through while the video plays.+ val parking = librespot != null &&+ (kind == PlayerKind.LOCAL || kind == PlayerKind.VIDEO)if (parking) {runCatching { player.playWhenReady = false }parkedSpotify = librespotlibrespot = nullplayerKind = kind- player = LocalPlayerFactory.create(playbackHost).also { built ->- built.playbackParameters = androidx.media3.common.PlaybackParameters(- AudioEffects.speed.value,- AudioEffects.pitch.value,- )- }+ player = buildPlayer(kind)session?.player = player// The queue on disk belongs to this player when it is the one being// restored into — leaving it out is what made a local song pauseddiff --git a/app/src/main/java/dev/lelonio/square/ui/MainViewModel.kt b/app/src/main/java/dev/lelonio/square/ui/MainViewModel.ktindex a04ac45..0c56f76 100644--- a/app/src/main/java/dev/lelonio/square/ui/MainViewModel.kt+++ b/app/src/main/java/dev/lelonio/square/ui/MainViewModel.kt@@ -1068,6 +1068,44 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {return id.takeIf { it.isNotEmpty() && it.all { c -> c.isLetterOrDigit() || c in "._-" } }}+ private val _videoFileId = MutableStateFlow<String?>(null)++ /**+ * The music video for what is playing, when there is one.+ *+ * Looked up in the catalogue rather than waited for: the account attaches a+ * video's id to a track only for the devices it believes can show one, and+ * it does not believe that of this app however it declares itself. Asking+ * works for anybody. See `catalog.rs`.+ */+ val videoFileId: StateFlow<String?> = _videoFileId.asStateFlow()++ private val _videoMode = MutableStateFlow(false)++ /** Whether the listener has asked to watch rather than listen. */+ val videoMode: StateFlow<Boolean> = _videoMode.asStateFlow()++ private var videoLookup: kotlinx.coroutines.Job? = null++ /** Asked once per track, and forgotten the moment the track changes. */+ fun lookUpVideo(trackUri: String?) {+ videoLookup?.cancel()+ _videoFileId.value = null+ _videoMode.value = false+ if (trackUri == null || !trackUri.startsWith("spotify:track:")) return+ videoLookup = viewModelScope.launch(Dispatchers.IO) {+ val answer = NativeBridge.trackVideo(trackUri) ?: return@launch+ val fileId = runCatching {+ org.json.JSONObject(answer).optString("fileId").takeIf { it.isNotEmpty() }+ }.getOrNull()+ _videoFileId.value = fileId+ }+ }++ fun toggleVideo() {+ _videoMode.value = !_videoMode.value && _videoFileId.value != null+ }+val recent: StateFlow<List<CatalogTrack>> =combine(container.recentStore.tracks, container.preferences.backend) { tracks, _ ->tracks.filter { container.activeBackend.owns(it.uri) }diff --git a/app/src/main/java/dev/lelonio/square/ui/SquareApp.kt b/app/src/main/java/dev/lelonio/square/ui/SquareApp.ktindex 1b729c3..dea228c 100644--- a/app/src/main/java/dev/lelonio/square/ui/SquareApp.kt+++ b/app/src/main/java/dev/lelonio/square/ui/SquareApp.kt@@ -316,7 +316,7 @@ fun SquareApp(val localPosition = rememberPositionMs(player, local.isPlaying)val remotePosition = rememberRemotePositionMs(remote)val positionMs = if (remote != null) remotePosition else localPosition- val queue by rememberQueue(player)+ val queueSource = rememberQueue(player)val videoOn by YouTubeVideoMode.enabled.collectAsStateWithLifecycle()// Video keeps playing when the app leaves the foreground, surface or no@@ -355,6 +355,50 @@ fun SquareApp(val addFriendState by viewModel.addFriendState.collectAsStateWithLifecycle()val followedArtists by viewModel.followedArtists.collectAsStateWithLifecycle()val savedAlbums by viewModel.savedAlbums.collectAsStateWithLifecycle()+ val videoFileId by viewModel.videoFileId.collectAsStateWithLifecycle()+ val spotifyVideoOn by dev.lelonio.square.backend.spotify.SpotifyVideoMode.enabled+ .collectAsStateWithLifecycle()+ val spotifyVideoGeneration by dev.lelonio.square.backend.spotify.SpotifyVideoMode.generation+ .collectAsStateWithLifecycle()++ val liveQueue by queueSource++ // What was queued before the video took over.+ //+ // A video is one item: the player that shows it has a timeline of one, and+ // the queue would read as empty for as long as it plays. The songs waiting+ // are still waiting — the engine holding them is parked, not stopped — so+ // the list keeps showing them rather than lying about it.+ var queuedBeforeVideo by remember {+ mutableStateOf<List<dev.lelonio.square.ui.player.QueueEntry>>(emptyList())+ }+ val queue = if (spotifyVideoOn) queuedBeforeVideo else liveQueue+ LaunchedEffect(spotifyVideoOn, liveQueue) {+ if (!spotifyVideoOn) queuedBeforeVideo = liveQueue+ }++ // And whether there is anywhere to skip to, which is the same question.+ //+ // The video's own player has one item and answers no to both, so the+ // buttons went grey over a queue that was still there. Skipping leaves the+ // video anyway — see SpotifyVideoMode.skip — so what they should show is+ // what the queue behind it can do.+ var skipsBeforeVideo by remember { mutableStateOf(false to false) }+ LaunchedEffect(spotifyVideoOn, playback.hasNext, playback.hasPrevious) {+ if (!spotifyVideoOn) skipsBeforeVideo = playback.hasNext to playback.hasPrevious+ }+ val playerState = if (spotifyVideoOn) {+ playback.copy(+ hasNext = skipsBeforeVideo.first,+ hasPrevious = skipsBeforeVideo.second,+ )+ } else {+ playback+ }++ // Asked once per track: almost no song has a video, and the answer is two+ // catalogue calls rather than something the account volunteers.+ LaunchedEffect(playback.mediaId) { viewModel.lookUpVideo(playback.mediaId) }val devices by viewModel.devices.collectAsStateWithLifecycle()val addToPlaylist by viewModel.addToPlaylist.collectAsStateWithLifecycle()val trackSort by viewModel.trackSort.collectAsStateWithLifecycle()@@ -584,6 +628,51 @@ fun SquareApp(return}+ // Turned on its side while a video plays, the phone is a screen.+ //+ // Nobody rotates a music player to read a queue sideways; they rotate it+ // because they are watching something. So landscape gives the picture the+ // whole display and nothing else, the way every video app does — and+ // turning back brings the player exactly as it was, since none of this+ // touches what is playing.+ val landscape = androidx.compose.ui.platform.LocalConfiguration.current.orientation ==+ android.content.res.Configuration.ORIENTATION_LANDSCAPE+ if (landscape && (videoOn || spotifyVideoOn) && player != null) {+ SquareTheme(seed = accent) {+ dev.lelonio.square.ui.player.FullScreenVideo(+ player = player,+ attachKey = spotifyVideoGeneration,+ state = playerState,+ positionMs = positionMs,+ onTogglePlay = { player.togglePlay() },+ onNext = {+ if (spotifyVideoOn) {+ dev.lelonio.square.backend.spotify.SpotifyVideoMode.skip(forward = true)+ } else {+ player.seekToNextMediaItem()+ }+ },+ onPrevious = {+ if (spotifyVideoOn) {+ dev.lelonio.square.backend.spotify.SpotifyVideoMode.skip(forward = false)+ } else {+ player.seekToPreviousMediaItem()+ }+ },+ onSeek = { player.seekTo(it) },+ onToggleShuffle = { player.shuffleModeEnabled = !player.shuffleModeEnabled },+ onCycleRepeat = {+ player.repeatMode = when (player.repeatMode) {+ Player.REPEAT_MODE_OFF -> Player.REPEAT_MODE_ALL+ Player.REPEAT_MODE_ALL -> Player.REPEAT_MODE_ONE+ else -> Player.REPEAT_MODE_OFF+ }+ },+ )+ }+ return+ }+SquareTheme(seed = accent) {// Material's default content colour is black, and it used to arrive from// the Surface that wrapped this tree. That Surface is gone — it painted@@ -1364,8 +1453,25 @@ fun SquareApp(dev.lelonio.square.ui.player.LocalGlassEnabled.current),) {PlayerScreen(- state = playback,+ state = playerState,positionMs = positionMs,+ videoFileId = videoFileId,+ videoMode = spotifyVideoOn,+ onToggleVideo = {+ // Handed to the service, which swaps what+ // the session is playing: the video carries+ // the song's own audio and answers the+ // transport controls. See SpotifyVideoMode.+ val at = player?.currentPosition ?: 0L+ val id = videoFileId+ if (spotifyVideoOn) {+ dev.lelonio.square.backend.spotify+ .SpotifyVideoMode.listen(at)+ } else if (id != null) {+ dev.lelonio.square.backend.spotify+ .SpotifyVideoMode.watch(id, at)+ }+ },onCollapse = { scope.launch { expand.animateTo(0f, expandSpec) } },onTogglePlay = {if (remote != null) {@@ -1380,11 +1486,24 @@ fun SquareApp(},onNext = {if (remote != null) onRemote(RemoteConnect::next)- else player?.seekToNextMediaItem()+ // Skipping is about the song: a video has+ // nowhere of its own to go, and the queue+ // is held by the engine parked behind it.+ else if (spotifyVideoOn) {+ dev.lelonio.square.backend.spotify+ .SpotifyVideoMode.skip(forward = true)+ } else {+ player?.seekToNextMediaItem()+ }},onPrevious = {if (remote != null) onRemote(RemoteConnect::previous)- else player?.seekToPreviousMediaItem()+ else if (spotifyVideoOn) {+ dev.lelonio.square.backend.spotify+ .SpotifyVideoMode.skip(forward = false)+ } else {+ player?.seekToPreviousMediaItem()+ }},onSeek = { target ->if (remote != null) {@@ -1512,8 +1631,9 @@ fun SquareApp(?.startsWith("ytmusic:track:") == true}?.let { { YouTubeVideoMode.toggle(it) } },- videoOn = videoOn,+ videoOn = videoOn || spotifyVideoOn,videoPlayer = player,+ videoAttachKey = spotifyVideoGeneration,)}},diff --git a/app/src/main/java/dev/lelonio/square/ui/player/FullScreenVideo.kt b/app/src/main/java/dev/lelonio/square/ui/player/FullScreenVideo.ktnew file mode 100644index 0000000..aeead97--- /dev/null+++ b/app/src/main/java/dev/lelonio/square/ui/player/FullScreenVideo.kt@@ -0,0 +1,243 @@+package dev.lelonio.square.ui.player++import androidx.compose.foundation.background+import androidx.compose.foundation.clickable+import androidx.compose.foundation.layout.Arrangement+import androidx.compose.foundation.layout.Box+import androidx.compose.foundation.layout.Column+import androidx.compose.foundation.layout.Row+import androidx.compose.foundation.layout.aspectRatio+import androidx.compose.foundation.layout.fillMaxSize+import androidx.compose.foundation.layout.fillMaxWidth+import androidx.compose.foundation.layout.padding+import androidx.compose.foundation.layout.size+import androidx.compose.material3.Icon+import androidx.compose.material3.MaterialTheme+import androidx.compose.material3.Slider+import androidx.compose.material3.SliderDefaults+import androidx.compose.material3.Text+import androidx.compose.runtime.Composable+import androidx.compose.runtime.DisposableEffect+import androidx.compose.runtime.State+import androidx.compose.runtime.getValue+import androidx.compose.runtime.mutableFloatStateOf+import androidx.compose.runtime.mutableStateOf+import androidx.compose.runtime.remember+import androidx.compose.runtime.setValue+import androidx.compose.ui.Alignment+import androidx.compose.ui.Modifier+import androidx.compose.ui.graphics.Color+import androidx.compose.ui.res.stringResource+import androidx.compose.ui.unit.dp+import androidx.compose.foundation.layout.Spacer+import androidx.compose.foundation.layout.height+import dev.lelonio.square.ui.glass.backdrop.backdrops.layerBackdrop+import dev.lelonio.square.ui.glass.backdrop.backdrops.rememberLayerBackdrop+import androidx.media3.common.Player+import androidx.media3.common.VideoSize+import com.adamglin.PhosphorIcons+import com.adamglin.phosphoricons.Fill+import com.adamglin.phosphoricons.fill.Pause+import com.adamglin.phosphoricons.fill.Play+import com.adamglin.phosphoricons.fill.SkipBack+import com.adamglin.phosphoricons.fill.SkipForward+import dev.lelonio.square.R+import dev.lelonio.square.ui.library.formatDuration++/**+ * The video, and nothing else, for a phone held on its side.+ *+ * Turning the screen while something is playing is how everybody asks for a+ * video to fill the display, so that is what it does. What it is not is a+ * second player: the same session is playing, and the controls here are the+ * same ones, so this is a way of looking at what is already happening.+ *+ * The controls are drawn over the picture and fade out of the way, because in+ * this orientation the picture is the whole point; a tap brings them back.+ */+@Composable+fun FullScreenVideo(+ player: Player,+ attachKey: Any?,+ state: PlaybackState,+ positionMs: State<Long>,+ onTogglePlay: () -> Unit,+ onNext: () -> Unit,+ onPrevious: () -> Unit,+ onSeek: (Long) -> Unit,+ onToggleShuffle: () -> Unit,+ onCycleRepeat: () -> Unit,+) {+ // The video's own shape, so it is fitted rather than stretched. A default+ // until the first frame reports one: the alternative is a black screen+ // that jumps as soon as the decoder speaks.+ var ratio by remember { mutableFloatStateOf(16f / 9f) }+ DisposableEffect(player) {+ val listener = object : Player.Listener {+ override fun onVideoSizeChanged(size: VideoSize) {+ if (size.width > 0 && size.height > 0) {+ ratio = size.width * size.pixelWidthHeightRatio / size.height+ }+ }+ }+ player.addListener(listener)+ onDispose { player.removeListener(listener) }+ }++ var showControls by remember { mutableStateOf(true) }++ // Out of the way on their own, the way every video player does it. Only+ // while something is playing: paused, the controls are what the listener+ // is looking for.+ androidx.compose.runtime.LaunchedEffect(showControls, state.isPlaying, positionMs.value / 4000) {+ if (showControls && state.isPlaying) {+ kotlinx.coroutines.delay(CONTROLS_LINGER_MS)+ showControls = false+ }+ }++ // What the glass refracts here is the black behind the picture, recorded on+ // its own: a pane cannot sample a protected video, and asking it to would+ // leave the discs empty.+ val backdrop = rememberLayerBackdrop()++ // The status bar and the gesture pill go away with everything else.+ //+ // A phone turned on its side to watch something is not showing a page any+ // more, and leaving the system's own furniture on top of the picture is the+ // difference between a video player and a video in a window. Put back on+ // the way out, whatever the way out was.+ val context = androidx.compose.ui.platform.LocalContext.current+ DisposableEffect(context) {+ val window = (context as? android.app.Activity)?.window+ val controller = window?.let {+ androidx.core.view.WindowCompat.getInsetsController(it, it.decorView)+ }+ controller?.systemBarsBehavior =+ androidx.core.view.WindowInsetsControllerCompat.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE+ controller?.hide(androidx.core.view.WindowInsetsCompat.Type.systemBars())+ onDispose {+ controller?.show(androidx.core.view.WindowInsetsCompat.Type.systemBars())+ }+ }++ Box(+ Modifier+ .fillMaxSize()+ .background(Color.Black)+ .clickable(interactionSource = null, indication = null) {+ showControls = !showControls+ },+ contentAlignment = Alignment.Center,+ ) {+ // The recorded layer holds the black and nothing else, and it has to be+ // a sibling of the controls rather than their parent: a pane drawn+ // inside the layer it samples asks the render tree to draw itself to+ // draw itself, and the stack runs out.+ Box(+ Modifier+ .fillMaxSize()+ .background(Color.Black)+ .layerBackdrop(backdrop),+ )++ VideoSurface(player, attachKey, Modifier.fillMaxWidth().aspectRatio(ratio))++ // A wash over the picture while the controls are up, so white text and+ // white glass have something to sit on. It goes with them.+ androidx.compose.animation.AnimatedVisibility(+ visible = showControls,+ enter = androidx.compose.animation.fadeIn(+ androidx.compose.animation.core.tween(180),+ ),+ exit = androidx.compose.animation.fadeOut(+ androidx.compose.animation.core.tween(220),+ ),+ modifier = Modifier.fillMaxSize(),+ ) {+ Box(+ Modifier+ .fillMaxSize()+ .background(Color.Black.copy(alpha = 0.35f)),+ )+ }++ // What is playing, at the top, with the controls: on a screen with no+ // cover and no page around it, this is the only thing naming the song.+ androidx.compose.animation.AnimatedVisibility(+ visible = showControls,+ enter = androidx.compose.animation.fadeIn(+ androidx.compose.animation.core.tween(180),+ ),+ exit = androidx.compose.animation.fadeOut(+ androidx.compose.animation.core.tween(220),+ ),+ modifier = Modifier.align(Alignment.TopStart),+ ) {+ Column(Modifier.padding(horizontal = 28.dp, vertical = 22.dp)) {+ Text(+ state.title,+ style = MaterialTheme.typography.titleMedium,+ color = Color.White,+ maxLines = 1,+ overflow = androidx.compose.ui.text.style.TextOverflow.Ellipsis,+ )+ Text(+ state.artist,+ style = MaterialTheme.typography.bodySmall,+ color = Color.White.copy(alpha = 0.75f),+ maxLines = 1,+ overflow = androidx.compose.ui.text.style.TextOverflow.Ellipsis,+ )+ }+ }++ androidx.compose.animation.AnimatedVisibility(+ visible = showControls,+ enter = androidx.compose.animation.fadeIn(+ androidx.compose.animation.core.tween(180),+ ),+ exit = androidx.compose.animation.fadeOut(+ androidx.compose.animation.core.tween(220),+ ),+ modifier = Modifier.align(Alignment.BottomCenter),+ ) {+ androidx.compose.runtime.CompositionLocalProvider(+ androidx.compose.material3.LocalContentColor provides Color.White,+ ) {+ Column(+ Modifier+ .fillMaxWidth()+ .padding(horizontal = 28.dp, vertical = 20.dp),+ ) {+ // The vertical player's own bar and its own discs of glass. Same+ // controls, same material: turning the phone changes where they sit+ // on the screen, not what the app is made of.+ GlassProgressBar(+ positionMs = positionMs,+ durationMs = state.durationMs,+ onSeek = onSeek,+ accentColor = Color.White,+ trackColor = Color.White.copy(alpha = 0.28f),+ )+ TimeRow(positionMs, state.durationMs)++ Spacer(Modifier.height(10.dp))++ Controls(+ state = state,+ backdrop = backdrop,+ onTogglePlay = onTogglePlay,+ onNext = onNext,+ onPrevious = onPrevious,+ onToggleShuffle = onToggleShuffle,+ onCycleRepeat = onCycleRepeat,+ )+ }+ }+ }+ }+}++/** How long the controls stay up before getting out of the picture. */+private const val CONTROLS_LINGER_MS = 3_500Ldiff --git a/app/src/main/java/dev/lelonio/square/ui/player/PlayerScreen.kt b/app/src/main/java/dev/lelonio/square/ui/player/PlayerScreen.ktindex 05145b1..3995af7 100644--- a/app/src/main/java/dev/lelonio/square/ui/player/PlayerScreen.kt+++ b/app/src/main/java/dev/lelonio/square/ui/player/PlayerScreen.kt@@ -73,6 +73,11 @@ import androidx.compose.ui.unit.dpimport androidx.media3.common.Playerimport androidx.media3.common.util.UnstableApiimport coil.compose.AsyncImage+import androidx.lifecycle.compose.collectAsStateWithLifecycle+import kotlinx.coroutines.withContext+import dev.lelonio.square.backend.spotify.panelAt+import androidx.core.graphics.scale+import dev.lelonio.square.ui.glass.pressableimport dev.lelonio.square.ui.glass.backdrop.Backdropimport dev.lelonio.square.ui.glass.backdrop.backdrops.layerBackdropimport dev.lelonio.square.ui.glass.backdrop.backdrops.rememberCombinedBackdrop@@ -83,6 +88,8 @@ import dev.lelonio.square.ui.components.Artworkimport dev.lelonio.square.ui.glass.LiquidButtonimport dev.lelonio.square.ui.library.formatDurationimport dev.lelonio.square.ui.theme.softShadow+import com.adamglin.phosphoricons.regular.MonitorPlay+import com.adamglin.phosphoricons.regular.MusicNotesimport com.adamglin.PhosphorIconsimport com.adamglin.phosphoricons.Fillimport com.adamglin.phosphoricons.Regular@@ -150,6 +157,11 @@ fun PlayerScreen(lyrics: dev.lelonio.square.data.Lyrics?,lyricsLoading: Boolean,onPlayQueueItem: (Int) -> Unit,+ /** The music video for this track, when the catalogue has one. */+ videoFileId: String? = null,+ /** Whether the listener has asked to watch rather than listen. */+ videoMode: Boolean = false,+ onToggleVideo: () -> Unit = {},reverb: Float,onSpeed: (Float) -> Unit,onPitch: (Float) -> Unit,@@ -225,6 +237,8 @@ fun PlayerScreen(videoOn: Boolean = false,/** The player to hang the video surface off; null when there is no video. */videoPlayer: Player? = null,+ /** Changes when the session starts playing a new video; see VideoStage. */+ videoAttachKey: Any? = null,/*** False when the source is not Spotify.*@@ -242,6 +256,14 @@ fun PlayerScreen(// for the cover. Written from the gesture and read only inside a// graphicsLayer, so following the finger costs a redraw of one layer rather// than a recomposition of the player.+ // No Canvas behind a video.+ //+ // The Canvas is the loop that stands in for a picture when there is none;+ // with the video playing there is one, and two moving images arguing over+ // the same screen is what the official client does not do either.+ @Suppress("NAME_SHADOWING")+ val canvas = canvas?.takeIf { !videoOn }+val canvasShift = remember { mutableFloatStateOf(0f) }// Whether the clip has put a frame on screen yet.@@ -316,7 +338,11 @@ fun PlayerScreen(// The stand-in for a Canvas, for the tracks that have none, and// only when it has been asked for. In the same layer as the light// above and for the same reason.- if (canvas == null) {+ // The stand-in is for a screen with nothing moving on it. With a+ // video playing there is something, and the glow above is taken+ // from its own frames: drawing the cover's aura over that would+ // paint the song's colours on top of the video's.+ if (canvas == null && !videoOn) {CoverAura(colors = auraColors, playing = state.isPlaying)}@@ -533,7 +559,11 @@ fun PlayerScreen(// the controller it holds is gone the moment// the activity stops.videoPlayer?.let {- VideoStage(it) { ambient = it }+ VideoStage(+ it,+ videoAttachKey,+ protectedContent = videoMode,+ ) { ambient = it }}}@@ -621,6 +651,46 @@ fun PlayerScreen(Spacer(Modifier.height(20.dp))+ // Only for the few tracks that have a video, and above+ // the title because that is where the official client+ // puts it: it is a choice about this song, not about+ // the app.+ if (videoFileId != null) {+ GlassSurface(+ backdrop = glassBackdrop,+ surfaceColor = GlassFilm,+ shape = RoundedCornerShape(50),+ modifier = Modifier+ .align(Alignment.CenterHorizontally)+ .pressable(onClick = onToggleVideo),+ ) {+ Row(+ Modifier.padding(horizontal = 16.dp, vertical = 10.dp),+ verticalAlignment = Alignment.CenterVertically,+ ) {+ // The icon says what the button leads to,+ // like the words beside it: a screen while+ // listening, a note while watching.+ Icon(+ if (videoMode) PhosphorIcons.Regular.MusicNotes+ else PhosphorIcons.Regular.MonitorPlay,+ contentDescription = null,+ modifier = Modifier+ .padding(end = 8.dp)+ .size(18.dp),+ )+ Text(+ stringResource(+ if (videoMode) R.string.switch_to_audio+ else R.string.switch_to_video,+ ),+ style = MaterialTheme.typography.labelLarge,+ )+ }+ }+ Spacer(Modifier.height(14.dp))+ }+// Title and artist on their own capsule, with the two// per-track actions on the right.GlassSurface(@@ -791,14 +861,46 @@ fun PlayerScreen(* back on the way out, or the player would go on rendering into a dead one.*/@Composable-private fun VideoStage(player: Player, onAmbient: (AmbientEdges) -> Unit) {+private fun VideoStage(+ player: Player,+ attachKey: Any?,+ /**+ * True for content the device decrypts in a protected buffer.+ *+ * Which decides the surface, and takes the ambient glow away with it: a+ * texture is the only surface whose pixels can be read back, and pixels+ * that can be read back are exactly what protected playback does not+ * allow. Drawn into one, a Widevine video shows nothing at all — the+ * decoder refuses the surface rather than the licence. So this gets the+ * plain surface the floating window uses, and the glow falls back to the+ * cover's colours, which is the honest best available.+ */+ protectedContent: Boolean,+ onAmbient: (AmbientEdges) -> Unit,+) {+ if (protectedContent) {+ VideoSurface(+ player,+ attachKey,+ Modifier+ .fillMaxWidth()+ .aspectRatio(16f / 9f)+ .clip(RoundedCornerShape(18.dp)),+ )+ PanelAmbient(player, onAmbient)+ return+ }+val context = androidx.compose.ui.platform.LocalContext.current// A TextureView rather than the SurfaceView the floating window uses: only// a texture can be read back, and reading the picture back is the whole// ambient effect — the glow is the video's own colour, not a guess made// from the cover.val texture = remember(context) { android.view.TextureView(context) }- DisposableEffect(player, texture) {+ // `attachKey` as well as the player: the session can change what it is+ // playing on without changing the handle this screen holds, and the+ // picture goes to whoever the surface was last given to.+ DisposableEffect(player, texture, attachKey) {player.setVideoTextureView(texture)onDispose { player.clearVideoTextureView(texture) }}@@ -926,12 +1028,73 @@ private const val AMBIENT_FADE_MS = 2500/** Side of the thumbnail each sample is taken from. */private const val AMBIENT_SAMPLE = 16+/**+ * The glow for a video whose own pixels cannot be read.+ *+ * Sampled from the thumbnail sheets the scrubber uses — ordinary JPEGs, one+ * frame a second — because protected playback decrypts into a buffer nothing+ * can read back. Same effect, same pictures, arrived at from the outside.+ *+ * One small fetch every couple of seconds, and the sheet holds thirty-six+ * frames, so most of those are already in the cache the loader keeps.+ */+@Composable+private fun PanelAmbient(player: Player, onAmbient: (AmbientEdges) -> Unit) {+ val manifest by dev.lelonio.square.backend.spotify.SpotifyVideoMode.manifest+ .collectAsStateWithLifecycle()+ val report by rememberUpdatedState(onAmbient)+ val http = remember { okhttp3.OkHttpClient() }++ LaunchedEffect(manifest) {+ val current = manifest ?: return@LaunchedEffect+ var lastUrl: String? = null+ var sheet: android.graphics.Bitmap? = null+ while (true) {+ val at = withContext(kotlinx.coroutines.Dispatchers.Main) { player.currentPosition }+ val panel = current.panelAt(at)+ if (panel == null) {+ kotlinx.coroutines.delay(AMBIENT_INTERVAL_MS)+ continue+ }+ withContext(kotlinx.coroutines.Dispatchers.IO) {+ runCatching {+ if (panel.url != lastUrl) {+ val request = okhttp3.Request.Builder().url(panel.url).build()+ val bytes = http.newCall(request).execute().use { it.body?.bytes() }+ sheet?.recycle()+ sheet = bytes?.let {+ android.graphics.BitmapFactory.decodeByteArray(it, 0, it.size)+ }+ lastUrl = panel.url+ }+ val whole = sheet ?: return@runCatching+ // The one frame out of the sheet, small: the glow is four+ // averaged edges, and averaging is cheaper on few pixels.+ val frame = android.graphics.Bitmap.createBitmap(+ whole,+ panel.left.coerceIn(0, (whole.width - 1).coerceAtLeast(0)),+ panel.top.coerceIn(0, (whole.height - 1).coerceAtLeast(0)),+ panel.width.coerceAtMost(whole.width - panel.left),+ panel.height.coerceAtMost(whole.height - panel.top),+ )+ val small = frame.scale(AMBIENT_SAMPLE, AMBIENT_SAMPLE)+ frame.recycle()+ val edges = small.edges()+ small.recycle()+ withContext(kotlinx.coroutines.Dispatchers.Main) { report(edges) }+ }+ }+ kotlinx.coroutines.delay(AMBIENT_INTERVAL_MS)+ }+ }+}+/** The surface itself, shared with the floating window; see [VideoStage]. */@Composable-fun VideoSurface(player: Player, modifier: Modifier = Modifier) {+fun VideoSurface(player: Player, attachKey: Any? = null, modifier: Modifier = Modifier) {val context = androidx.compose.ui.platform.LocalContext.currentval surface = remember(context) { android.view.SurfaceView(context) }- DisposableEffect(player, surface) {+ DisposableEffect(player, surface, attachKey) {player.setVideoSurfaceView(surface)onDispose { player.clearVideoSurfaceView(surface) }}@@ -1210,7 +1373,7 @@ private fun Modifier.plainClickable(onClick: () -> Unit): Modifier {/** Isolated so the ticking position recomposes only these two labels. */@Composable-private fun TimeRow(positionMs: State<Long>, durationMs: Long) {+internal fun TimeRow(positionMs: State<Long>, durationMs: Long) {Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.SpaceBetween) {Text(formatDuration(positionMs.value),@@ -1226,7 +1389,7 @@ private fun TimeRow(positionMs: State<Long>, durationMs: Long) {}@Composable-private fun Controls(+internal fun Controls(state: PlaybackState,backdrop: Backdrop,onTogglePlay: () -> Unit,diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xmlindex ebfcbb3..7ef7729 100644--- a/app/src/main/res/values-de/strings.xml+++ b/app/src/main/res/values-de/strings.xml@@ -40,6 +40,8 @@<string name="settings">Einstellungen</string><string name="home">Start</string><string name="library">Bibliothek</string>+ <string name="switch_to_video">Zum Video wechseln</string>+ <string name="switch_to_audio">Nur Audio</string><string name="library_all">Alles</string><string name="search">Suchen</string><string name="now_playing">Läuft gerade</string>diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xmlindex f59631c..e90fd79 100644--- a/app/src/main/res/values-es/strings.xml+++ b/app/src/main/res/values-es/strings.xml@@ -40,6 +40,8 @@<string name="settings">Ajustes</string><string name="home">Inicio</string><string name="library">Biblioteca</string>+ <string name="switch_to_video">Cambiar a vídeo</string>+ <string name="switch_to_audio">Cambiar a solo audio</string><string name="library_all">Todo</string><string name="search">Buscar</string><string name="now_playing">Reproduciendo</string>diff --git a/app/src/main/res/values-fr/strings.xml b/app/src/main/res/values-fr/strings.xmlindex 10dbb81..20b6a8f 100644--- a/app/src/main/res/values-fr/strings.xml+++ b/app/src/main/res/values-fr/strings.xml@@ -40,6 +40,8 @@<string name="settings">Réglages</string><string name="home">Accueil</string><string name="library">Bibliothèque</string>+ <string name="switch_to_video">Passer à la vidéo</string>+ <string name="switch_to_audio">Passer à l\'audio seul</string><string name="library_all">Tout</string><string name="search">Rechercher</string><string name="now_playing">En cours de lecture</string>diff --git a/app/src/main/res/values-hi/strings.xml b/app/src/main/res/values-hi/strings.xmlindex d15b75b..f5e70d4 100644--- a/app/src/main/res/values-hi/strings.xml+++ b/app/src/main/res/values-hi/strings.xml@@ -45,6 +45,8 @@<string name="settings">सेटिंग</string><string name="home">होम</string><string name="library">लाइब्रेरी</string>+ <string name="switch_to_video">वीडियो पर जाएँ</string>+ <string name="switch_to_audio">केवल ऑडियो पर जाएँ</string><string name="library_all">सब कुछ</string><string name="search">खोजें</string><string name="now_playing">अभी चल रहा है</string>diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xmlindex 9b40ba8..5dc41fa 100644--- a/app/src/main/res/values-it/strings.xml+++ b/app/src/main/res/values-it/strings.xml@@ -41,6 +41,8 @@<string name="settings">Impostazioni</string><string name="home">Home</string><string name="library">Libreria</string>+ <string name="switch_to_video">Passa al video</string>+ <string name="switch_to_audio">Passa a solo audio</string><string name="library_all">Tutto</string><string name="search">Cerca</string><string name="now_playing">In riproduzione</string>diff --git a/app/src/main/res/values-pt/strings.xml b/app/src/main/res/values-pt/strings.xmlindex c45e051..e0b9067 100644--- a/app/src/main/res/values-pt/strings.xml+++ b/app/src/main/res/values-pt/strings.xml@@ -40,6 +40,8 @@<string name="settings">Configurações</string><string name="home">Início</string><string name="library">Biblioteca</string>+ <string name="switch_to_video">Mudar para vídeo</string>+ <string name="switch_to_audio">Mudar para só áudio</string><string name="library_all">Tudo</string><string name="search">Buscar</string><string name="now_playing">Tocando agora</string>diff --git a/app/src/main/res/values-tr/strings.xml b/app/src/main/res/values-tr/strings.xmlindex 880be80..b8f005b 100644--- a/app/src/main/res/values-tr/strings.xml+++ b/app/src/main/res/values-tr/strings.xml@@ -45,6 +45,8 @@<string name="settings">Ayarlar</string><string name="home">Ana sayfa</string><string name="library">Kitaplık</string>+ <string name="switch_to_video">Videoya geç</string>+ <string name="switch_to_audio">Yalnızca sese geç</string><string name="library_all">Tümü</string><string name="search">Ara</string><string name="now_playing">Şimdi çalıyor</string>diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xmlindex 9151cfe..4df6f4d 100644--- a/app/src/main/res/values/strings.xml+++ b/app/src/main/res/values/strings.xml@@ -51,6 +51,8 @@<string name="settings">Settings</string><string name="home">Home</string><string name="library">Library</string>+ <string name="switch_to_video">Switch to video</string>+ <string name="switch_to_audio">Switch to audio</string><string name="library_all">Everything</string><string name="search">Search</string><string name="now_playing">Now playing</string>diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.tomlindex 84a10d1..83fc540 100644--- a/gradle/libs.versions.toml+++ b/gradle/libs.versions.toml@@ -31,6 +31,7 @@ media3-session = { module = "androidx.media3:media3-session", version.ref = "med# The Canvas clip, and every note of the YouTube Music backend. Spotify audio# still comes from the librespot engine instead.media3-exoplayer = { module = "androidx.media3:media3-exoplayer", version.ref = "media3" }+media3-exoplayer-dash = { module = "androidx.media3:media3-exoplayer-dash", version.ref = "media3" }media3-ui = { module = "androidx.media3:media3-ui", version.ref = "media3" }kotlinx-coroutines-android = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-android", version.ref = "coroutines" }diff --git a/native/src/catalog.rs b/native/src/catalog.rsindex 737e24f..bbdd3b0 100644--- a/native/src/catalog.rs+++ b/native/src/catalog.rs@@ -532,6 +532,150 @@ fn parse_canvas(bytes: &[u8]) -> Option<String> {// --- Minimal protobuf wire helpers -----------------------------------------+/// An access token minted by the engine's own session.+///+/// The app has one of its own from signing in, but that one lapses while the+/// engine goes on working: playback holds the credential the access point+/// issued and never needs the OAuth half. Anything that talks to spclient+/// outside the engine — the video manifest, the licence — should ask here+/// instead, or it stops working on exactly the days playback does not.+pub fn access_token() -> EngineResult<String> {+ let session = with_session(|s| s.clone())?;+ block_on(async move {+ session+ .login5()+ .auth_token()+ .await+ .map(|token| token.access_token)+ .map_err(|e| format!("no access token: {e}"))+ })+}++/// The music video a track has, or `null`.+///+/// Two questions to the catalogue, because the answer lives in two places.+/// `VIDEO_ASSOCIATIONS` on an ordinary track names the *other* track — the+/// variant of the same song that is the video — and `ORIGINAL_VIDEO` on that+/// one carries the file id the video manifest is addressed by.+///+/// Asked here rather than read from the player state, which is where the+/// official client takes it from: the account only attaches it for a device it+/// believes can show video, and it does not believe that of this one whatever+/// it declares about itself. The catalogue answers anybody who asks.+///+/// Hand-encoded like the canvas request below, and read the same way: the+/// request is three fields deep and the answer is one field inside an `Any`,+/// so writing the bytes out keeps both shapes visible.+///+/// BatchedEntityRequest { repeated EntityRequest entity_request = 2 }+/// EntityRequest { string entity_uri = 1, repeated ExtensionQuery query = 2 }+/// ExtensionQuery { ExtensionKind extension_kind = 1 }+pub fn track_video(track_uri: &str) -> EngineResult<String> {+ let session = with_session(|s| s.clone())?;+ let uri = track_uri.to_string();++ block_on(async move {+ let associated = match extension(&session, &uri, VIDEO_ASSOCIATIONS).await? {+ Some(bytes) => uris_in(&bytes)+ .into_iter()+ .find(|found| found != &uri)+ .ok_or_else(|| String::new()),+ None => Err(String::new()),+ };++ // No association is the ordinary answer: almost no track has a video.+ let Ok(video_uri) = associated else {+ return Ok("null".to_string());+ };++ let Some(bytes) = extension(&session, &video_uri, ORIGINAL_VIDEO).await? else {+ return Ok("null".to_string());+ };++ match file_id_in(&bytes) {+ Some(file_id) => Ok(json!({ "trackUri": video_uri, "fileId": file_id }).to_string()),+ None => Ok("null".to_string()),+ }+ })+}++/// `VIDEO_ASSOCIATIONS`, which names a song's video as another track.+const VIDEO_ASSOCIATIONS: u64 = 99;++/// `ORIGINAL_VIDEO`, which carries the file id that addresses the video itself.+const ORIGINAL_VIDEO: u64 = 85;++/// One extended-metadata question about one entity.+async fn extension(+ session: &Session,+ entity_uri: &str,+ kind: u64,+) -> Result<Option<Vec<u8>>, String> {+ let mut query = Vec::new();+ write_varint(&mut query, 1 << 3);+ write_varint(&mut query, kind);++ let mut entity = Vec::new();+ write_string_field(&mut entity, 1, entity_uri);+ write_len_delimited(&mut entity, 2, &query);++ let mut body = Vec::new();+ write_len_delimited(&mut body, 2, &entity);++ let bytes = session+ .spclient()+ .request(+ &Method::POST,+ "/extended-metadata/v0/extended-metadata",+ None,+ Some(&body),+ )+ .await+ .map_err(|e| format!("extended metadata failed: {e}"))?;++ Ok(Some(bytes.to_vec()).filter(|bytes| !bytes.is_empty()))+}++/// Every `spotify:track:` uri in a response, whatever it is nested inside.+///+/// Scanned rather than parsed: the reply wraps the extension's own message in+/// an `Any`, and the uri appears at a depth that changes with the extension.+/// What is being looked for is unmistakable on sight.+fn uris_in(bytes: &[u8]) -> Vec<String> {+ let needle = b"spotify:track:";+ let mut found = Vec::new();+ for start in 0..bytes.len() {+ if bytes[start..].starts_with(needle) {+ let end = bytes[start..]+ .iter()+ .position(|byte| !(0x20..0x7f).contains(byte))+ .map(|offset| start + offset)+ .unwrap_or(bytes.len());+ if let Ok(uri) = std::str::from_utf8(&bytes[start..end]) {+ found.push(uri.to_string());+ }+ }+ }+ found+}++/// The sixteen bytes of a file id, hex encoded.+///+/// `OriginalVideo { bytes file_id = 1 }`, inside the `Any` the extension+/// replies with: the tail of the message is `0a 10` and then the id, which is+/// the one length-delimited sixteen-byte field in the whole answer.+fn file_id_in(bytes: &[u8]) -> Option<String> {+ let position = bytes+ .windows(2)+ .rposition(|window| window == [0x0a, 0x10])?;+ let start = position + 2;+ let end = start + 16;+ if end > bytes.len() {+ return None;+ }+ Some(bytes[start..end].iter().map(|b| format!("{b:02x}")).collect())+}+fn write_varint(out: &mut Vec<u8>, mut value: u64) {loop {let byte = (value & 0x7f) as u8;diff --git a/native/src/engine.rs b/native/src/engine.rsindex 0c4a3d1..a0ed811 100644--- a/native/src/engine.rs+++ b/native/src/engine.rs@@ -1586,11 +1586,16 @@ pub fn playing_here() -> EngineResult<String> {let playing = with_bundle(|bundle| bundle.spirc.playing())?;let escape = |value: &str| serde_json::to_string(value).unwrap_or_else(|_| "\"\"".into());let tracks: Vec<String> = playing.tracks.iter().map(|uri| escape(uri)).collect();+ // `videoId` rides along rather than getting a call of its own: it changes+ // exactly when the track does, and this is already the answer to "what is+ // this device playing". Empty for everything without a music video, which+ // is almost everything.Ok(format!(- "{{\"contextUri\":{},\"trackUri\":{},\"index\":{},\"tracks\":[{}]}}",+ "{{\"contextUri\":{},\"trackUri\":{},\"index\":{},\"videoId\":{},\"tracks\":[{}]}}",escape(&playing.context_uri),escape(&playing.track_uri),playing.index,+ escape(&playing.video_id),tracks.join(","),))}diff --git a/native/src/ffi.rs b/native/src/ffi.rsindex 4100a8a..55f0895 100644--- a/native/src/ffi.rs+++ b/native/src/ffi.rs@@ -361,6 +361,30 @@ pub extern "system" fn Java_dev_lelonio_square_nativecore_NativeBridge_nativeCon}+#[no_mangle]+pub extern "system" fn Java_dev_lelonio_square_nativecore_NativeBridge_nativeAccessToken(+ mut env: JNIEnv,+ _class: JClass,+) -> jstring {+ guard_string(&mut env, "AccessToken", catalog::access_token)+}++#[no_mangle]+pub extern "system" fn Java_dev_lelonio_square_nativecore_NativeBridge_nativeTrackVideo(+ mut env: JNIEnv,+ _class: JClass,+ uri: JString,+) -> jstring {+ let uri = match read_string(&mut env, &uri) {+ Ok(value) => value,+ Err(message) => {+ let _ = env.throw_new(EXCEPTION, message);+ return JObject::null().into_raw() as jstring;+ }+ };+ guard_string(&mut env, "TrackVideo", || catalog::track_video(&uri))+}+#[no_mangle]pub extern "system" fn Java_dev_lelonio_square_nativecore_NativeBridge_nativeFriendActivity(mut env: JNIEnv,diff --git a/native/vendor/librespot-connect/src/spirc.rs b/native/vendor/librespot-connect/src/spirc.rsindex f124445..148380d 100644--- a/native/vendor/librespot-connect/src/spirc.rs+++ b/native/vendor/librespot-connect/src/spirc.rs@@ -195,6 +195,13 @@ pub struct PlayingHere {pub tracks: Vec<String>,/// Where the current track sits in `tracks`.pub index: usize,+ /// LOCAL PATCH: the video this track has, when it has one.+ ///+ /// Empty for almost everything. A track with a music video carries the id+ /// of it in the metadata the account sends with the track — there is no+ /// endpoint to ask, and no way to work it out from the track's own uri —+ /// so this is the only place a client can learn that a video exists.+ pub video_id: String,}impl Spirc {@@ -537,11 +544,18 @@ impl SpircTask {// taken, or the position would point past them.tracks.retain(|uri| uri.starts_with("spotify:track:"));let index = tracks.iter().position(|uri| *uri == track).unwrap_or(0);+ let video_id = state+ .track+ .as_ref()+ .and_then(|track| track.metadata.get("media.manifest_id"))+ .cloned()+ .unwrap_or_default();*playing = PlayingHere {context_uri: state.context_uri.clone(),track_uri: track,tracks,index,+ video_id,};}}