Skip to content

Unofficial YouTube Music / Spotify Premium client for Android, built in Liquid Glass. Audio effects, Spotify Connect, listening history, 8 languages. Rust core on librespot.

9.3 KB
Raw Blame
package dev.lelonio.square.playback

import androidx.annotation.OptIn
import androidx.media3.common.C
import androidx.media3.common.PlaybackParameters
import androidx.media3.common.audio.AudioProcessor
import androidx.media3.common.audio.BaseAudioProcessor
import androidx.media3.common.util.UnstableApi
import androidx.media3.common.util.Util
import androidx.media3.exoplayer.audio.DefaultAudioSink
import java.nio.ByteBuffer

/**
 * Speed and pitch for the players that are not the Spotify engine.
 *
 * Spotify's audio goes through the app's own sink, where a phase vocoder does
 * this work; a file on the phone goes through ExoPlayer, which has its own
 * stretcher built for speech. On music that one smears every attack and adds a
 * metallic edge, which is exactly what the same track sounds like on the two
 * paths at the same setting. This puts the same vocoder in ExoPlayer's chain,
 * so a song is altered the same way whatever it was read from.
 *
 * Sixteen-bit PCM only. Everything the decoders here hand over is that, and a
 * format this cannot take is passed through rather than refused: an untouched
 * song is a much better answer than silence.
 */
@OptIn(UnstableApi::class)
class BungeeAudioProcessor : BaseAudioProcessor() {

    private var stretcher: Stretcher? = null
    private var speed = 1f
    private var pitch = 1f

    /** Logged once rather than per packet, which would be several a second. */
    private var failureReported = false

    /**
     * Set from the sink's chain when the listener moves either slider.
     *
     * The stretcher takes them per packet, so nothing has to be rebuilt: what
     * is playing changes on the next few milliseconds of audio.
     */
    fun setSpeedAndPitch(speed: Float, pitch: Float) {
        this.speed = speed
        this.pitch = pitch
    }

    val isAltering: Boolean get() = speed != 1f || pitch != 1f

    override fun onConfigure(
        inputAudioFormat: AudioProcessor.AudioFormat,
    ): AudioProcessor.AudioFormat {
        if (inputAudioFormat.encoding != C.ENCODING_PCM_16BIT) {
            throw AudioProcessor.UnhandledAudioFormatException(inputAudioFormat)
        }
        return inputAudioFormat
    }

    override fun onFlush() {
        // Kept, deliberately.
        //
        // The sink flushes its processors whenever the parameters change, and
        // throwing the vocoder away here meant the next packet arrived at a
        // stretcher with an empty window: it has to fill one before it can
        // resynthesise anything, and that silence is the hole heard when the
        // speed slider moves. What it carries over instead is a few tens of
        // milliseconds of the audio before the change, which after a seek is
        // the only cost — and one nobody can hear.
        //
        // The format is a different matter: a stretcher is built for a sample
        // rate and a channel count, and a track with either different needs a
        // new one. That is what [stretcher] checks.
    }

    override fun onReset() {
        stretcher?.release()
        stretcher = null
        failureReported = false
    }

    override fun queueInput(inputBuffer: ByteBuffer) {
        val size = inputBuffer.remaining()
        if (size == 0) return

        // Nothing to do at rest, and doing nothing is worth the check: this
        // runs on the audio thread for every packet of every track.
        if (!isAltering) {
            replaceOutputBuffer(size).put(inputBuffer).flip()
            return
        }

        val frames = size / (inputAudioFormat.channelCount * BYTES_PER_SAMPLE)
        val stretcher = stretcher(frames)
        val produced = stretcher?.process(inputBuffer, size, speed, pitch)
        if (produced == null) {
            if (!failureReported) {
                failureReported = true
                android.util.Log.w(TAG, "stretcher unavailable, playing untouched")
            }
            // Position is already past what was read on the failed attempt for
            // a partial read; take what is left rather than dropping the packet.
            val remaining = inputBuffer.remaining()
            if (remaining > 0) replaceOutputBuffer(remaining).put(inputBuffer).flip()
            return
        }

        replaceOutputBuffer(produced.remaining()).put(produced).flip()
    }

    /** One stretcher per format, sized for the largest packet seen so far. */
    private fun stretcher(frames: Int): Stretcher? {
        val rate = inputAudioFormat.sampleRate
        val channels = inputAudioFormat.channelCount
        val existing = stretcher
        if (existing != null && frames <= maxFrames && rate == builtForRate &&
            channels == builtForChannels
        ) {
            return existing
        }

        existing?.release()
        maxFrames = maxOf(frames, maxFrames, MIN_PACKET_FRAMES)
        builtForRate = rate
        builtForChannels = channels
        return Stretcher.create(rate, channels, maxFrames).also { stretcher = it }
    }

    private var maxFrames = 0
    private var builtForRate = 0
    private var builtForChannels = 0

    private companion object {
        const val TAG = "SquareStretch"
        const val BYTES_PER_SAMPLE = 2

        /** A decoder's packet, with room to spare, so the first one fits. */
        const val MIN_PACKET_FRAMES = 8192
    }
}

/**
 * The sink's chain, with the vocoder in it instead of the platform's stretcher.
 *
 * ExoPlayer asks the chain to apply speed and pitch, and asks it afterwards how
 * long the audio it has played was in media time. Both answers have to come
 * from whoever is doing the stretching, or the position on screen runs at a
 * different rate from the sound.
 */
/**
 * The same vocal removal, for the players that are not the Spotify engine.
 *
 * In the chain before the stretcher, because the trick rests on what the two
 * channels have in common and a resynthesised pair no longer has the same two.
 * See [CentreExtractor].
 */
@OptIn(UnstableApi::class)
class VocalAudioProcessor : BaseAudioProcessor() {

    private val vocals = CentreExtractor()

    override fun onConfigure(
        inputAudioFormat: AudioProcessor.AudioFormat,
    ): AudioProcessor.AudioFormat {
        if (inputAudioFormat.encoding != C.ENCODING_PCM_16BIT) {
            throw AudioProcessor.UnhandledAudioFormatException(inputAudioFormat)
        }
        return inputAudioFormat
    }

    override fun queueInput(inputBuffer: ByteBuffer) {
        val size = inputBuffer.remaining()
        if (size == 0) return

        val amount = AudioEffects.karaoke.value
        val out = replaceOutputBuffer(size)
        val start = out.position()
        out.put(inputBuffer)
        out.position(start)
        if (amount > 0f) {
            vocals.apply(
                out,
                size,
                inputAudioFormat.channelCount,
                inputAudioFormat.sampleRate,
                amount,
            )
        }
        out.position(start)
        out.limit(start + size)
    }
}

/**
 * The equaliser, for the players that are not the Spotify engine.
 *
 * After the stretcher, so the bands sit where they were set whatever the speed
 * and pitch: before it, a pitch shift would carry the bass boost up with the
 * bass. See [Equalizer].
 */
@OptIn(UnstableApi::class)
class EqualizerAudioProcessor : BaseAudioProcessor() {

    private val filter = EqualizerFilter()

    override fun onConfigure(
        inputAudioFormat: AudioProcessor.AudioFormat,
    ): AudioProcessor.AudioFormat {
        if (inputAudioFormat.encoding != C.ENCODING_PCM_16BIT) {
            throw AudioProcessor.UnhandledAudioFormatException(inputAudioFormat)
        }
        return inputAudioFormat
    }

    override fun queueInput(inputBuffer: ByteBuffer) {
        val size = inputBuffer.remaining()
        if (size == 0) return
        val out = replaceOutputBuffer(size)
        val start = out.position()
        out.put(inputBuffer)
        out.position(start)
        filter.apply(out, size, inputAudioFormat.channelCount, inputAudioFormat.sampleRate)
        out.position(start)
        out.limit(start + size)
    }
}

@OptIn(UnstableApi::class)
class BungeeProcessorChain(
    private val bungee: BungeeAudioProcessor = BungeeAudioProcessor(),
    /** Run after the vocoder, at speed one: YouTube Music's resampling to its mixer. */
    private val after: Array<AudioProcessor> = emptyArray(),
) : DefaultAudioSink.AudioProcessorChain {

    private var parameters = PlaybackParameters.DEFAULT

    private val vocals = VocalAudioProcessor()

    private val equalizer = EqualizerAudioProcessor()

    override fun getAudioProcessors(): Array<AudioProcessor> = arrayOf(vocals, bungee, equalizer, *after)

    override fun applyPlaybackParameters(
        playbackParameters: PlaybackParameters,
    ): PlaybackParameters {
        parameters = playbackParameters
        bungee.setSpeedAndPitch(playbackParameters.speed, playbackParameters.pitch)
        return playbackParameters
    }

    override fun applySkipSilenceEnabled(skipSilenceEnabled: Boolean): Boolean = false

    /** How much media a stretch of playing covers, which is the speed itself. */
    override fun getMediaDuration(playoutDuration: Long): Long =
        Util.getMediaDurationForPlayoutDuration(playoutDuration, parameters.speed)

    /** Nothing is dropped here; silence is skipped by nobody in this chain. */
    override fun getSkippedOutputFrameCount(): Long = 0
}