Skip to content

High performance self-hosted photo and video management solution.

fix(mobile): apply exif orientation to remote raw photos on android (#29906)

Santo Shakil2cdbd0d00f95
diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt
index 2f5db1ca0e..7b49d8ca67 100644
--- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt
+++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt
@@ -64,13 +64,7 @@ fun Bitmap.toNativeBuffer(): Map<String, Long> {
// native convert declined (OOM/lock) -> fall through to the Skia copy path below.
}
// Other non-8888 configs (e.g. HDR F16) still need Skia's convert; 8-bit is copied as-is.
- val bitmap = if (config != Bitmap.Config.ARGB_8888) {
- val converted = copy(Bitmap.Config.ARGB_8888, false)
- recycle()
- converted ?: throw IOException("could not convert bitmap to ARGB_8888")
- } else {
- this
- }
+ val bitmap = toArgb8888()
val size = bitmap.width * bitmap.height * 4
val pointer = NativeBuffer.allocate(size)
try {
@@ -217,7 +211,7 @@ class LocalImagesImpl(context: Context) : LocalImageApi {
if (orientation == ExifInterface.ORIENTATION_NORMAL || orientation == ExifInterface.ORIENTATION_UNDEFINED) {
bitmap.toNativeBuffer()
} else {
- rotateToNativeBuffer(bitmap, orientation, signal)
+ rotateToNativeBuffer(bitmap, orientation)
}
}
// Don't re-check cancellation here: res owns a malloc'd buffer, and bailing to CANCELLED would
@@ -257,39 +251,7 @@ class LocalImagesImpl(context: Context) : LocalImageApi {
}
private fun rawOrientation(uri: Uri): Int {
- return resolver.openInputStream(uri)?.use {
- ExifInterface(it).getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL)
- } ?: ExifInterface.ORIENTATION_NORMAL
- }
-
- // ImageDecoder / loadThumbnail skip EXIF orientation for raw (e.g. DNG) on Q+, so the decoded
- // bitmap comes back unrotated. Rotate it into the output buffer in native code (one pass, no
- // intermediate rotated bitmap).
- private fun rotateToNativeBuffer(bitmap: Bitmap, orientation: Int, signal: CancellationSignal): Map<String, Long> {
- signal.throwIfCanceled()
- // Force ARGB_8888: the native rotate needs a lockable 8888 buffer, and toNativeBuffer() below
- // allocates width*height*4 (an F16/HDR decode would otherwise under-allocate). No-op for the
- // common already-8888 case.
- val src = if (bitmap.config != Bitmap.Config.ARGB_8888) {
- val converted = bitmap.copy(Bitmap.Config.ARGB_8888, false)
- bitmap.recycle()
- converted ?: throw IOException("could not convert bitmap to ARGB_8888")
- } else {
- bitmap
- }
- try {
- val info = IntArray(3)
- val pointer = NativeImage.rotate(src, orientation, info)
- if (pointer == 0L) throw IOException("native rotate failed for orientation $orientation")
- return mapOf(
- "pointer" to pointer,
- "width" to info[0].toLong(),
- "height" to info[1].toLong(),
- "rowBytes" to info[2].toLong()
- )
- } finally {
- if (!src.isRecycled) src.recycle()
- }
+ return resolver.openInputStream(uri)?.use { readOrientation(it) } ?: ExifInterface.ORIENTATION_NORMAL
}
private fun decodeVideoThumbnail(id: Long, target: Size, signal: CancellationSignal): Bitmap {
diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RawOrientation.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RawOrientation.kt
new file mode 100644
index 0000000000..e91ff871b3
--- /dev/null
+++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RawOrientation.kt
@@ -0,0 +1,68 @@
+package app.alextran.immich.images
+
+import android.graphics.Bitmap
+import androidx.exifinterface.media.ExifInterface
+import app.alextran.immich.NativeImage
+import java.io.IOException
+import java.io.InputStream
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+
+fun readOrientation(input: InputStream): Int =
+ ExifInterface(input).getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL)
+
+// Reads the EXIF orientation (TIFF tag 0x0112) from a raw file's IFD0.
+fun readRawOrientation(buffer: ByteBuffer, len: Int): Int {
+ try {
+ if (len < 8) return ExifInterface.ORIENTATION_NORMAL
+ buffer.order(
+ when (buffer.get(0).toInt() and 0xFF) {
+ 0x49 -> ByteOrder.LITTLE_ENDIAN
+ 0x4D -> ByteOrder.BIG_ENDIAN
+ else -> return ExifInterface.ORIENTATION_NORMAL
+ }
+ )
+ val ifd0 = buffer.getInt(4).toLong() and 0xFFFFFFFFL
+ if (ifd0 < 8 || ifd0 + 2 > len) return ExifInterface.ORIENTATION_NORMAL
+ val start = ifd0.toInt()
+ val entries = buffer.getShort(start).toInt() and 0xFFFF
+ if (entries <= 0 || entries > 512) return ExifInterface.ORIENTATION_NORMAL
+ // Each IFD entry is 12 bytes: tag(2) type(2) count(4) value(4); orientation is a SHORT in value.
+ for (i in 0 until entries) {
+ val p = start + 2 + i * 12
+ if (p + 12 > len) return ExifInterface.ORIENTATION_NORMAL
+ if ((buffer.getShort(p).toInt() and 0xFFFF) == 0x0112) {
+ val value = buffer.getShort(p + 8).toInt() and 0xFFFF
+ return if (value in 1..8) value else ExifInterface.ORIENTATION_NORMAL
+ }
+ }
+ return ExifInterface.ORIENTATION_NORMAL
+ } catch (_: Exception) {
+ return ExifInterface.ORIENTATION_NORMAL
+ }
+}
+
+fun Bitmap.toArgb8888(): Bitmap {
+ if (config == Bitmap.Config.ARGB_8888) return this
+ val converted = copy(Bitmap.Config.ARGB_8888, false)
+ recycle()
+ return converted ?: throw IOException("could not convert bitmap to ARGB_8888")
+}
+
+// Force ARGB_8888 first: the native rotate needs a lockable 8888 buffer and allocates w*h*4.
+fun rotateToNativeBuffer(bitmap: Bitmap, orientation: Int): Map<String, Long> {
+ val src = bitmap.toArgb8888()
+ try {
+ val info = IntArray(3)
+ val pointer = NativeImage.rotate(src, orientation, info)
+ if (pointer == 0L) throw IOException("native rotate failed for orientation $orientation")
+ return mapOf(
+ "pointer" to pointer,
+ "width" to info[0].toLong(),
+ "height" to info[1].toLong(),
+ "rowBytes" to info[2].toLong()
+ )
+ } finally {
+ if (!src.isRecycled) src.recycle()
+ }
+}
diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImagesImpl.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImagesImpl.kt
index 226305dd85..1623c9cb09 100644
--- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImagesImpl.kt
+++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImagesImpl.kt
@@ -5,6 +5,7 @@ import android.graphics.ImageDecoder
import android.os.Build
import android.os.CancellationSignal
import android.os.OperationCanceledException
+import androidx.exifinterface.media.ExifInterface
import app.alextran.immich.INITIAL_BUFFER_SIZE
import app.alextran.immich.NativeBuffer
import app.alextran.immich.NativeByteBuffer
@@ -30,6 +31,35 @@ private const val MAX_PREALLOC_BYTES = 128 * 1024 * 1024
private class RemoteRequest(val cancellationSignal: CancellationSignal)
+// The full mimeTypes.raw set the server can serve (short + vendor forms).
+private val RAW_MIME_TYPES = setOf(
+ "image/3fr", "image/ari", "image/arw",
+ "image/cap", "image/cin", "image/cr2",
+ "image/cr3", "image/crw", "image/dcr",
+ "image/dng", "image/erf", "image/fff",
+ "image/iiq", "image/k25", "image/kdc",
+ "image/mrw", "image/nef", "image/nrw",
+ "image/orf", "image/ori", "image/pef",
+ "image/psd", "image/raf", "image/raw",
+ "image/rw2", "image/rwl", "image/sr2",
+ "image/srf", "image/srw", "image/vnd.adobe.photoshop",
+ "image/x-adobe-dng", "image/x-arriflex-ari", "image/x-canon-cr2",
+ "image/x-canon-cr3", "image/x-canon-crw", "image/x-epson-erf",
+ "image/x-fuji-raf", "image/x-hasselblad-3fr", "image/x-hasselblad-fff",
+ "image/x-kodak-dcr", "image/x-kodak-k25", "image/x-kodak-kdc",
+ "image/x-leica-rwl", "image/x-minolta-mrw", "image/x-nikon-nef",
+ "image/x-nikon-nrw", "image/x-olympus-orf", "image/x-olympus-ori",
+ "image/x-panasonic-raw", "image/x-panasonic-rw2", "image/x-pentax-pef",
+ "image/x-phantom-cin", "image/x-phaseone-cap", "image/x-phaseone-iiq",
+ "image/x-samsung-srw", "image/x-sigma-x3f", "image/x-sony-arw",
+ "image/x-sony-sr2", "image/x-sony-srf", "image/x3f",
+)
+
+private fun isRawMime(contentType: String?): Boolean {
+ val mime = contentType?.substringBefore(';')?.trim()?.lowercase() ?: return false
+ return mime in RAW_MIME_TYPES
+}
+
class RemoteImagesImpl(context: Context) : RemoteImageApi {
private val requestMap = ConcurrentHashMap<Long, RemoteRequest>()
@@ -57,7 +87,7 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi {
ImageFetcherManager.fetch(
url,
signal,
- onSuccess = { buffer ->
+ onSuccess = { buffer, contentType ->
if (signal.isCanceled) {
requestMap.remove(requestId)
buffer.free()
@@ -72,8 +102,18 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi {
if (!preferEncoded && Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
decodeExecutor.execute {
val res = if (signal.isCanceled) null else try {
- val source = ImageDecoder.createSource(NativeBuffer.wrap(buffer.pointer, buffer.offset))
- source.decodeBitmap().toNativeBuffer()
+ val bitmap = ImageDecoder.createSource(NativeBuffer.wrap(buffer.pointer, buffer.offset)).decodeBitmap()
+ // The embedded preview a raw decodes to has no orientation, so read the container's.
+ val orientation = if (isRawMime(contentType)) {
+ readRawOrientation(NativeBuffer.wrap(buffer.pointer, buffer.offset), buffer.offset)
+ } else {
+ ExifInterface.ORIENTATION_NORMAL
+ }
+ if (orientation == ExifInterface.ORIENTATION_NORMAL || orientation == ExifInterface.ORIENTATION_UNDEFINED) {
+ bitmap.toNativeBuffer()
+ } else {
+ rotateToNativeBuffer(bitmap, orientation)
+ }
} catch (_: Throwable) {
null
}
@@ -154,7 +194,7 @@ private object ImageFetcherManager {
fun fetch(
url: String,
signal: CancellationSignal,
- onSuccess: (NativeByteBuffer) -> Unit,
+ onSuccess: (NativeByteBuffer, String?) -> Unit,
onFailure: (Exception) -> Unit,
) {
fetcher.fetch(url, signal, onSuccess, onFailure)
@@ -185,7 +225,7 @@ private sealed interface ImageFetcher {
fun fetch(
url: String,
signal: CancellationSignal,
- onSuccess: (NativeByteBuffer) -> Unit,
+ onSuccess: (NativeByteBuffer, String?) -> Unit,
onFailure: (Exception) -> Unit,
)
@@ -203,7 +243,7 @@ private class CronetImageFetcher : ImageFetcher {
override fun fetch(
url: String,
signal: CancellationSignal,
- onSuccess: (NativeByteBuffer) -> Unit,
+ onSuccess: (NativeByteBuffer, String?) -> Unit,
onFailure: (Exception) -> Unit,
) {
synchronized(stateLock) {
@@ -271,7 +311,7 @@ private class CronetImageFetcher : ImageFetcher {
}
private class FetchCallback(
- private val onSuccess: (NativeByteBuffer) -> Unit,
+ private val onSuccess: (NativeByteBuffer, String?) -> Unit,
private val onFailure: (Exception) -> Unit,
private val onComplete: () -> Unit,
) : UrlRequest.Callback() {
@@ -328,7 +368,9 @@ private class CronetImageFetcher : ImageFetcher {
}
override fun onSucceeded(request: UrlRequest, info: UrlResponseInfo) {
- onSuccess(buffer!!)
+ val contentType = info.allHeaders.entries
+ .firstOrNull { it.key.equals("content-type", ignoreCase = true) }?.value?.firstOrNull()
+ onSuccess(buffer!!, contentType)
onComplete()
}
@@ -379,7 +421,7 @@ private class OkHttpImageFetcher private constructor(
override fun fetch(
url: String,
signal: CancellationSignal,
- onSuccess: (NativeByteBuffer) -> Unit,
+ onSuccess: (NativeByteBuffer, String?) -> Unit,
onFailure: (Exception) -> Unit,
) {
synchronized(stateLock) {
@@ -433,7 +475,7 @@ private class OkHttpImageFetcher private constructor(
buffer.ensureHeadroom()
}
}
- onSuccess(buffer)
+ onSuccess(buffer, response.header("Content-Type"))
} catch (e: Exception) {
buffer.free()
onFailure(e)