A cross platform Audiobookshelf client for Android and iOS
Dart
Stop transcriptions and clips starting half a minute before the bookmark on MP3 files
Barnabasd8ded782491f
diff --git a/android/app/src/main/kotlin/com/barnabas/absorb/AudioClipExporter.kt b/android/app/src/main/kotlin/com/barnabas/absorb/AudioClipExporter.ktindex 6199e6ed..66a22165 100644--- a/android/app/src/main/kotlin/com/barnabas/absorb/AudioClipExporter.kt+++ b/android/app/src/main/kotlin/com/barnabas/absorb/AudioClipExporter.kt@@ -39,8 +39,19 @@ object AudioClipExporter {durationSeconds: Double,outPath: String,): Boolean {+ // An MP3 can't be seeked accurately (see [Mp3Slicer]), so cut the window+ // out by frame count first and decode that instead. Anything else - an+ // M4B book, a streamed URL - goes straight through untouched.+ val sliced = if (isLocal) Mp3Slicer.prepare(context, source, startSeconds, durationSeconds) else nullreturn try {- val decoded = decodeWindow(context, source, isLocal, headers, startSeconds, durationSeconds)+ val decoded = decodeWindow(+ context,+ sliced?.path ?: source,+ isLocal,+ headers,+ sliced?.startSeconds ?: startSeconds,+ durationSeconds,+ )if (decoded == null || decoded.pcm.isEmpty()) {Log.e(TAG, "no PCM decoded from clip window")return false@@ -52,6 +63,8 @@ object AudioClipExporter {Log.e(TAG, "exportM4a failed: ${e.message}", e)try { File(outPath).delete() } catch (_: Exception) {}false+ } finally {+ sliced?.temp?.delete()}}@@ -93,7 +106,10 @@ object AudioClipExporter {val startUs = (startSeconds * 1_000_000L).toLong()val durationUs = (durationSeconds * 1_000_000L).toLong()- extractor.seekTo(startUs, MediaExtractor.SEEK_TO_CLOSEST_SYNC)+ val landedUs = AudioSeek.seekTo(extractor, startUs)+ if (landedUs in 0 until startUs - 1_000_000L) {+ Log.d(TAG, "seek landed ${(startUs - landedUs) / 1000}ms early, trimming the head")+ }codec = MediaCodec.createDecoderByType(mime)codec.configure(inFormat, null, null, 0)@@ -110,7 +126,6 @@ object AudioClipExporter {inFormat.getInteger(MediaFormat.KEY_PCM_ENCODING) else 2val out = ByteArrayOutputStream()- var firstSampleUs = -1Lvar collectedUs = 0Lwhile (!sawOutputEOS) {@@ -138,9 +153,14 @@ object AudioClipExporter {} else if (outIndex >= 0) {val outBuf = codec.getOutputBuffer(outIndex)if (outBuf != null && info.size > 0) {- if (firstSampleUs < 0) firstSampleUs = info.presentationTimeUs- appendInterleaved(outBuf, info, outChannels, target, pcmEncoding, out)- collectedUs = info.presentationTimeUs - firstSampleUs+ // Decoding starts on a frame boundary at or before the+ // requested time, so drop whatever came out ahead of it -+ // otherwise the clip starts early and runs short.+ val aheadUs = startUs - info.presentationTimeUs+ val skipFrames =+ if (aheadUs > 0) (aheadUs * outRate / 1_000_000L).toInt() else 0+ appendInterleaved(outBuf, info, outChannels, target, pcmEncoding, out, skipFrames)+ collectedUs = out.size().toLong() * 1_000_000L / (outRate.toLong() * target * 2)}codec.releaseOutputBuffer(outIndex, false)if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) sawOutputEOS = true@@ -155,7 +175,11 @@ object AudioClipExporter {}}- /** Append decoded frames as 16-bit little-endian PCM with [target] channels. */+ /**+ * Append decoded frames as 16-bit little-endian PCM with [target] channels.+ * [skipFrames] drops that many frames off the front, for a buffer that+ * starts before the clip the caller asked for.+ */private fun appendInterleaved(buf: ByteBuffer,info: MediaCodec.BufferInfo,@@ -163,6 +187,7 @@ object AudioClipExporter {target: Int,pcmEncoding: Int,out: ByteArrayOutputStream,+ skipFrames: Int = 0,) {buf.position(info.offset)buf.limit(info.offset + info.size)@@ -170,16 +195,20 @@ object AudioClipExporter {if (pcmEncoding == 4) { // ENCODING_PCM_FLOATval fb = buf.order(ByteOrder.nativeOrder()).asFloatBuffer()val frames = fb.remaining() / ch+ val skip = skipFrames.coerceIn(0, frames)+ fb.position(skip * ch)val frame = FloatArray(ch)- for (f in 0 until frames) {+ for (f in skip until frames) {for (c in 0 until ch) frame[c] = fb.get()writeFrame16(out, frame, ch, target)}} else { // 16-bit PCMval sb = buf.order(ByteOrder.LITTLE_ENDIAN).asShortBuffer()val frames = sb.remaining() / ch+ val skip = skipFrames.coerceIn(0, frames)+ sb.position(skip * ch)val frame = FloatArray(ch)- for (f in 0 until frames) {+ for (f in skip until frames) {for (c in 0 until ch) frame[c] = sb.get() / 32768fwriteFrame16(out, frame, ch, target)}diff --git a/android/app/src/main/kotlin/com/barnabas/absorb/AudioSeek.kt b/android/app/src/main/kotlin/com/barnabas/absorb/AudioSeek.ktnew file mode 100644index 00000000..9a01c1bd--- /dev/null+++ b/android/app/src/main/kotlin/com/barnabas/absorb/AudioSeek.kt@@ -0,0 +1,67 @@+package com.barnabas.absorb++import android.media.MediaExtractor++/**+ * Seeking an MP3 is guesswork on Android: MediaExtractor works from the bitrate+ * or a Xing table, so on a variable-bitrate file - which most podcasts are - it+ * can land tens of seconds away from the time it was asked for. An MP4/M4B has a+ * real sample table and lands where you point it, which is why this only started+ * to matter once podcast episodes could be bookmarked.+ *+ * Walking frames forward from an undershoot costs nothing but header parsing, so+ * the fix is to seek short on purpose and then step in.+ */+internal object AudioSeek {+ /** Decoders need a little audio before the target to come up to speed. */+ private const val PRE_ROLL_US = 200_000L++ /** How far back to restart from when a seek overshoots the target. */+ private const val BACK_OFF_US = 30_000_000L++ /** Stops a malformed file from spinning the walk forever. */+ private const val MAX_ADVANCE = 200_000++ /** Within this much of the start, decode from zero instead of seeking. */+ private const val NEAR_START_US = 1_000_000L++ /**+ * Leave [extractor] on the frame just before [targetUs], and return the+ * presentation time it actually landed on (-1 if the file ended first).+ * Callers still have to trim the decoded head: the landing frame starts up+ * to [PRE_ROLL_US] early, and nothing here decodes anything.+ */+ fun seekTo(extractor: MediaExtractor, targetUs: Long): Long {+ // Close enough to the start to just decode from it. Worth doing rather+ // than seeking, because timestamps counted from zero are honest even on+ // an MP3 - so the caller's head trim lands on the exact frame. This is+ // the path an [Mp3Slicer] slice takes, its window being half a second in.+ if (targetUs <= NEAR_START_US) {+ extractor.seekTo(0L, MediaExtractor.SEEK_TO_PREVIOUS_SYNC)+ return 0L+ }+ extractor.seekTo(targetUs, MediaExtractor.SEEK_TO_PREVIOUS_SYNC)+ if (extractor.sampleTime > targetUs) {+ // Overshot even asking for the previous sync point, so the estimate+ // ran long. Restart further back and walk in from there.+ extractor.seekTo(+ (targetUs - BACK_OFF_US).coerceAtLeast(0L),+ MediaExtractor.SEEK_TO_PREVIOUS_SYNC,+ )+ }++ val limitUs = (targetUs - PRE_ROLL_US).coerceAtLeast(0L)+ var steps = 0+ while (steps++ < MAX_ADVANCE) {+ val t = extractor.sampleTime+ if (t < 0) break // ran off the end+ if (t >= limitUs) return t+ if (!extractor.advance()) break+ }++ // Ended up somewhere unusable - fall back to the plain seek so the+ // caller still gets audio, even if it starts in the wrong place.+ extractor.seekTo(targetUs, MediaExtractor.SEEK_TO_PREVIOUS_SYNC)+ return extractor.sampleTime+ }+}diff --git a/android/app/src/main/kotlin/com/barnabas/absorb/AudioWindowExtractor.kt b/android/app/src/main/kotlin/com/barnabas/absorb/AudioWindowExtractor.ktindex a9b275c4..50aeacaf 100644--- a/android/app/src/main/kotlin/com/barnabas/absorb/AudioWindowExtractor.kt+++ b/android/app/src/main/kotlin/com/barnabas/absorb/AudioWindowExtractor.kt@@ -39,6 +39,30 @@ object AudioWindowExtractor {startSeconds: Double,durationSeconds: Double,outPath: String,+ ): Boolean {+ // An MP3 can't be seeked accurately (see [Mp3Slicer]), so cut the window+ // out by frame count first and decode that instead. Anything else - an+ // M4B book, a streamed URL - goes straight through untouched.+ val sliced = Mp3Slicer.prepare(context, sourcePath, startSeconds, durationSeconds)+ return try {+ decodeToWav(+ context,+ sliced?.path ?: sourcePath,+ sliced?.startSeconds ?: startSeconds,+ durationSeconds,+ outPath,+ )+ } finally {+ sliced?.temp?.delete()+ }+ }++ private fun decodeToWav(+ context: Context,+ sourcePath: String,+ startSeconds: Double,+ durationSeconds: Double,+ outPath: String,): Boolean {val extractor = MediaExtractor()var codec: MediaCodec? = null@@ -64,7 +88,11 @@ object AudioWindowExtractor {val startUs = (startSeconds * 1_000_000L).toLong()val durationUs = (durationSeconds * 1_000_000L).toLong()- extractor.seekTo(startUs, MediaExtractor.SEEK_TO_CLOSEST_SYNC)+ val landedUs = AudioSeek.seekTo(extractor, startUs)+ // Logged every time, not just when it drifts: on an MP3 the landing+ // time is the extractor's own estimate, so "landed == want" here can+ // still mean the audio is somewhere else entirely.+ Log.d(TAG, "seek want=${startUs / 1000}ms landed=${landedUs / 1000}ms mime=$mime")codec = MediaCodec.createDecoderByType(mime)codec.configure(inputFormat, null, null, 0)@@ -80,7 +108,6 @@ object AudioWindowExtractor {inputFormat.getInteger(MediaFormat.KEY_PCM_ENCODING) else 2val mono = FloatBuf((srcRate * durationSeconds).toInt())- var firstSampleUs = -1Lvar collectedUs = 0Lwhile (!sawOutputEOS) {@@ -108,9 +135,15 @@ object AudioWindowExtractor {} else if (outIndex >= 0) {val outBuf = codec.getOutputBuffer(outIndex)if (outBuf != null && info.size > 0) {- if (firstSampleUs < 0) firstSampleUs = info.presentationTimeUs- appendMono(outBuf, info, outChannels, pcmEncoding, mono)- collectedUs = info.presentationTimeUs - firstSampleUs+ // Decoding starts on a frame boundary at or before the+ // requested time, so drop whatever came out ahead of it -+ // otherwise the window runs early and the bookmarked+ // moment falls off the end of it (or out of it entirely).+ val aheadUs = startUs - info.presentationTimeUs+ val skipFrames =+ if (aheadUs > 0) (aheadUs * outRate / 1_000_000L).toInt() else 0+ appendMono(outBuf, info, outChannels, pcmEncoding, mono, skipFrames)+ collectedUs = mono.size.toLong() * 1_000_000L / outRate}codec.releaseOutputBuffer(outIndex, false)if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) sawOutputEOS = true@@ -144,15 +177,28 @@ object AudioWindowExtractor {return -1}- /** Append decoded frames, downmixing all channels to a single mono float. */- private fun appendMono(buf: ByteBuffer, info: MediaCodec.BufferInfo, channels: Int, pcmEncoding: Int, out: FloatBuf) {+ /**+ * Append decoded frames, downmixing all channels to a single mono float.+ * [skipFrames] drops that many frames off the front, for a buffer that+ * starts before the window the caller asked for.+ */+ private fun appendMono(+ buf: ByteBuffer,+ info: MediaCodec.BufferInfo,+ channels: Int,+ pcmEncoding: Int,+ out: FloatBuf,+ skipFrames: Int = 0,+ ) {buf.position(info.offset)buf.limit(info.offset + info.size)val ch = if (channels <= 0) 1 else channelsif (pcmEncoding == 4) { // ENCODING_PCM_FLOATval fb = buf.order(ByteOrder.nativeOrder()).asFloatBuffer()val frames = fb.remaining() / ch- for (f in 0 until frames) {+ val skip = skipFrames.coerceIn(0, frames)+ fb.position(skip * ch)+ for (f in skip until frames) {var sum = 0ffor (c in 0 until ch) sum += fb.get()out.add(sum / ch)@@ -160,7 +206,9 @@ object AudioWindowExtractor {} else { // 16-bit PCMval sb = buf.order(ByteOrder.LITTLE_ENDIAN).asShortBuffer()val frames = sb.remaining() / ch- for (f in 0 until frames) {+ val skip = skipFrames.coerceIn(0, frames)+ sb.position(skip * ch)+ for (f in skip until frames) {var sum = 0for (c in 0 until ch) sum += sb.get().toInt()out.add((sum.toFloat() / ch) / 32768f)diff --git a/android/app/src/main/kotlin/com/barnabas/absorb/Mp3Slicer.kt b/android/app/src/main/kotlin/com/barnabas/absorb/Mp3Slicer.ktnew file mode 100644index 00000000..0da0a5f7--- /dev/null+++ b/android/app/src/main/kotlin/com/barnabas/absorb/Mp3Slicer.kt@@ -0,0 +1,292 @@+package com.barnabas.absorb++import android.content.Context+import android.net.Uri+import android.util.Log+import java.io.BufferedInputStream+import java.io.File+import java.io.FileOutputStream+import java.io.InputStream+import java.io.OutputStream++/**+ * Cuts a byte-exact slice out of an MP3 by counting frame headers from the start+ * of the file.+ *+ * MediaExtractor can't be asked to do this. An MP3 carries no seek table, so its+ * seek is a bitrate guess that lands tens of seconds out on a variable-bitrate+ * file - which most podcasts are - and it then reports the time it was asked for+ * rather than the time it reached, so the drift is invisible from the outside.+ * Frame headers each carry their own size, so walking them is exact, and only the+ * 4-byte headers have to be read: the frame bodies are skipped until the window+ * we actually want. Scanning half an hour of audio that way costs a sequential+ * read, not a decode.+ *+ * An MP4/M4B has a real sample table and seeks exactly, so books never come+ * through here.+ */+internal object Mp3Slicer {+ private const val TAG = "AbsorbTranscribe"++ /** Audio kept ahead of the requested time, to give the decoder a run-up. */+ private const val PRE_ROLL_S = 0.5++ /** How far into the file to look for the first frame before giving up. */+ private const val SYNC_SEARCH_BYTES = 128 * 1024++ /** Frames to walk before deciding the file isn't what its header claims. */+ private const val MAX_FRAMES = 5_000_000++ // Layer III bitrates in kbps. Index 0 is "free format" and 15 is invalid;+ // both are left as 0 so the frame is rejected rather than guessed at.+ private val BITRATES_V1 =+ intArrayOf(0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0)+ private val BITRATES_V2 =+ intArrayOf(0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0)+ private val RATES_V1 = intArrayOf(44100, 48000, 32000, 0)+ private val RATES_V2 = intArrayOf(22050, 24000, 16000, 0)+ private val RATES_V25 = intArrayOf(11025, 12000, 8000, 0)++ /**+ * A cut-down copy of the source. [startSeconds] is where the caller's+ * requested time sits inside it, and [temp] is the caller's to delete.+ */+ class Prepared(val path: String, val startSeconds: Double, val temp: File)++ private class Frame(val length: Int, val durationS: Double)++ /**+ * Slice [source] down to the frames covering [startSeconds] and the+ * [durationSeconds] after it. Returns null when the source isn't a Layer III+ * MP3 that walks cleanly, in which case the caller should carry on with the+ * original source untouched.+ */+ fun prepare(+ context: Context,+ source: String,+ startSeconds: Double,+ durationSeconds: Double,+ ): Prepared? {+ // The start of the file needs no help: a seek to ~0 lands on ~0.+ if (startSeconds <= PRE_ROLL_S) return null+ val temp = try {+ File.createTempFile("mp3slice", ".mp3", context.cacheDir)+ } catch (e: Exception) {+ Log.d(TAG, "mp3 slice skipped, no temp file: ${e.message}")+ return null+ }+ val prepared = try {+ scan(context, source, startSeconds, durationSeconds, temp)+ } catch (e: Exception) {+ Log.d(TAG, "mp3 slice skipped: ${e.message}")+ null+ }+ if (prepared == null) temp.delete()+ return prepared+ }++ private fun scan(+ context: Context,+ source: String,+ startSeconds: Double,+ durationSeconds: Double,+ temp: File,+ ): Prepared? {+ val raw = openStream(context, source) ?: return null+ val input = BufferedInputStream(raw, 64 * 1024)+ var out: FileOutputStream? = null+ try {+ if (!skipId3(input)) return null++ val header = ByteArray(4)+ if (!findFirstFrame(input, header)) return null++ val copyBuf = ByteArray(8192)+ val wantStart = startSeconds - PRE_ROLL_S+ val wantEnd = startSeconds + durationSeconds + PRE_ROLL_S+ var timeS = 0.0+ var sliceStartS = -1.0+ var haveHeader = true+ var frames = 0++ while (frames++ < MAX_FRAMES) {+ if (!haveHeader && !readFully(input, header, 4)) break+ haveHeader = false+ // Files often end with an ID3v1 or APE trailer, which is not a+ // frame. Once the slice is underway that just means the audio+ // ran out; before then, it means this isn't a file we can walk.+ val frame = parseHeader(header)+ if (frame == null || frame.length <= 4) {+ if (out != null) break+ return null+ }+ val body = frame.length - 4++ if (sliceStartS < 0 && timeS + frame.durationS > wantStart) {+ sliceStartS = timeS+ out = FileOutputStream(temp)+ }+ val sink = out+ if (sink != null) {+ sink.write(header)+ if (!copyFully(input, sink, body, copyBuf)) break+ } else if (!skipFully(input, body.toLong())) {+ break+ }+ timeS += frame.durationS+ if (sink != null && timeS >= wantEnd) break+ }++ out?.flush()+ // Never reached the window - the file ended early, or it isn't as+ // long as the caller thinks. Leave the old path to deal with it.+ if (out == null || sliceStartS < 0) return null++ val offset = startSeconds - sliceStartS+ Log.d(+ TAG,+ "mp3 slice: want=${"%.1f".format(startSeconds)}s " ++ "slice=${"%.1f".format(sliceStartS)}s offset=${"%.2f".format(offset)}s " ++ "frames=$frames bytes=${temp.length()}",+ )+ return Prepared(temp.absolutePath, offset, temp)+ } finally {+ try { out?.close() } catch (_: Exception) {}+ try { input.close() } catch (_: Exception) {}+ }+ }++ private fun openStream(context: Context, source: String): InputStream? = when {+ source.startsWith("content://") ->+ context.contentResolver.openInputStream(Uri.parse(source))+ // A streamed book is an http(s) URL - nothing to walk locally.+ source.startsWith("http") -> null+ else -> File(source).takeIf { it.isFile }?.inputStream()+ }++ /** Step over an ID3v2 tag if the file opens with one. */+ private fun skipId3(input: BufferedInputStream): Boolean {+ input.mark(16)+ val head = ByteArray(10)+ if (!readFully(input, head, 10)) return false+ val isId3 = head[0] == 'I'.code.toByte() &&+ head[1] == 'D'.code.toByte() &&+ head[2] == '3'.code.toByte()+ if (!isId3) {+ input.reset() // those 10 bytes were audio+ return true+ }+ // Syncsafe: seven bits per byte, top bit always clear.+ val size = ((head[6].toInt() and 0x7F) shl 21) or+ ((head[7].toInt() and 0x7F) shl 14) or+ ((head[8].toInt() and 0x7F) shl 7) or+ (head[9].toInt() and 0x7F)+ val footer = if ((head[5].toInt() and 0x10) != 0) 10L else 0L+ return skipFully(input, size.toLong() + footer)+ }++ /**+ * Leave [input] just past the first frame header, with that header in+ * [header]. A lone valid-looking header can turn up in leftover junk, so a+ * candidate only counts when the frame it describes is followed by another+ * valid header.+ */+ private fun findFirstFrame(input: BufferedInputStream, header: ByteArray): Boolean {+ if (!readFully(input, header, 4)) return false+ var searched = 0+ while (searched < SYNC_SEARCH_BYTES) {+ val frame = parseHeader(header)+ if (frame != null && nextHeaderFollows(input, frame)) return true+ // Shift the window on by a byte and try again.+ header[0] = header[1]+ header[1] = header[2]+ header[2] = header[3]+ val next = input.read()+ if (next < 0) return false+ header[3] = next.toByte()+ searched+++ }+ return false+ }++ /** Peek past this frame to check another header starts where it should. */+ private fun nextHeaderFollows(input: BufferedInputStream, frame: Frame): Boolean {+ val body = frame.length - 4+ if (body <= 0) return false+ input.mark(frame.length + 8)+ return try {+ if (!skipFully(input, body.toLong())) return false+ val peek = ByteArray(4)+ if (!readFully(input, peek, 4)) return false+ parseHeader(peek) != null+ } catch (_: Exception) {+ false+ } finally {+ try { input.reset() } catch (_: Exception) {}+ }+ }++ private fun parseHeader(h: ByteArray): Frame? {+ if ((h[0].toInt() and 0xFF) != 0xFF) return null+ val b1 = h[1].toInt() and 0xFF+ if ((b1 and 0xE0) != 0xE0) return null+ val version = (b1 shr 3) and 0x03 // 0 = MPEG2.5, 1 = reserved, 2 = MPEG2, 3 = MPEG1+ val layer = (b1 shr 1) and 0x03 // 1 = Layer III+ if (version == 1 || layer != 1) return null+ val b2 = h[2].toInt() and 0xFF+ val bitrate = (if (version == 3) BITRATES_V1 else BITRATES_V2)[(b2 shr 4) and 0x0F]+ val rateIndex = (b2 shr 2) and 0x03+ val rate = when (version) {+ 3 -> RATES_V1[rateIndex]+ 2 -> RATES_V2[rateIndex]+ else -> RATES_V25[rateIndex]+ }+ if (bitrate <= 0 || rate <= 0) return null+ val padding = (b2 shr 1) and 0x01+ val samples = if (version == 3) 1152 else 576+ val length = samples / 8 * bitrate * 1000 / rate + padding+ if (length < 24) return null+ return Frame(length, samples.toDouble() / rate)+ }++ private fun readFully(input: InputStream, buf: ByteArray, len: Int): Boolean {+ var off = 0+ while (off < len) {+ val n = input.read(buf, off, len - off)+ if (n < 0) return false+ off += n+ }+ return true+ }++ private fun skipFully(input: InputStream, count: Long): Boolean {+ var left = count+ while (left > 0) {+ val n = input.skip(left)+ if (n > 0) {+ left -= n+ } else {+ // skip() is allowed to do nothing; read a byte to make progress.+ if (input.read() < 0) return false+ left--+ }+ }+ return true+ }++ private fun copyFully(+ input: InputStream,+ out: OutputStream,+ count: Int,+ buf: ByteArray,+ ): Boolean {+ var left = count+ while (left > 0) {+ val n = input.read(buf, 0, minOf(left, buf.size))+ if (n < 0) return false+ out.write(buf, 0, n)+ left -= n+ }+ return true+ }+}diff --git a/pubspec.yaml b/pubspec.yamlindex dfa0acef..d36ee897 100644--- a/pubspec.yaml+++ b/pubspec.yaml@@ -1,7 +1,7 @@name: absorbdescription: An Audiobookshelf Clientpublish_to: 'none'-version: 1.9.3+247+version: 1.9.3+248environment:sdk: '>=3.10.0 <4.0.0'