Skip to content

High performance self-hosted photo and video management solution.

fix(mobile): apply exif orientation to android raw photos (#29337)

Santo Shakilaf2efda31057
diff --git a/mobile/android/app/CMakeLists.txt b/mobile/android/app/CMakeLists.txt
index 133bde4fc0..9cad7b915a 100644
--- a/mobile/android/app/CMakeLists.txt
+++ b/mobile/android/app/CMakeLists.txt
@@ -7,6 +7,7 @@ project(native_buffer LANGUAGES C)
add_library(native_buffer SHARED
src/main/cpp/native_buffer.c
+ src/main/cpp/native_image.c
)
target_link_libraries(native_buffer jnigraphics)
diff --git a/mobile/android/app/src/main/cpp/native_image.c b/mobile/android/app/src/main/cpp/native_image.c
new file mode 100644
index 0000000000..07bb7e9a2c
--- /dev/null
+++ b/mobile/android/app/src/main/cpp/native_image.c
@@ -0,0 +1,109 @@
+#include <jni.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <android/bitmap.h>
+
+// Cache-friendly block size for the tiled rotation (in pixels). 32x32 uint32 = 4KB, fits L1.
+#define TILE 32
+
+// EXIF orientation values (androidx.exifinterface.media.ExifInterface.ORIENTATION_*).
+enum {
+ ORIENTATION_FLIP_HORIZONTAL = 2,
+ ORIENTATION_ROTATE_180 = 3,
+ ORIENTATION_FLIP_VERTICAL = 4,
+ ORIENTATION_TRANSPOSE = 5,
+ ORIENTATION_ROTATE_90 = 6,
+ ORIENTATION_TRANSVERSE = 7,
+ ORIENTATION_ROTATE_270 = 8,
+};
+
+// The orientations that swap width and height. Must stay in sync with affine_for's dim usage.
+static int swaps_dims(int o) {
+ return o == ORIENTATION_ROTATE_90 || o == ORIENTATION_ROTATE_270 ||
+ o == ORIENTATION_TRANSPOSE || o == ORIENTATION_TRANSVERSE;
+}
+
+// A source pixel (sx, sy) maps to destination index base + sx*stepX + sy*stepY, where dw is the
+// destination width. This affine form covers all 8 EXIF orientations and matches the pixel layout
+// of Bitmap.createBitmap(src, matrixForExifOrientation(o)). int64_t so it stays correct on
+// armeabi-v7a (32-bit long) regardless of how large MAX_RAW_DECODE_PIXELS grows.
+static void affine_for(int o, int sw, int sh, int dw, int64_t *base, int64_t *stepX, int64_t *stepY) {
+ switch (o) {
+ case ORIENTATION_ROTATE_90: *base = sh - 1; *stepX = dw; *stepY = -1; break;
+ case ORIENTATION_ROTATE_270: *base = (int64_t) (sw - 1) * dw; *stepX = -dw; *stepY = 1; break;
+ case ORIENTATION_ROTATE_180: *base = (int64_t) (sh - 1) * dw + (sw - 1); *stepX = -1; *stepY = -dw; break;
+ case ORIENTATION_FLIP_HORIZONTAL: *base = sw - 1; *stepX = -1; *stepY = dw; break;
+ case ORIENTATION_FLIP_VERTICAL: *base = (int64_t) (sh - 1) * dw; *stepX = 1; *stepY = -dw; break;
+ case ORIENTATION_TRANSPOSE: *base = 0; *stepX = dw; *stepY = 1; break;
+ case ORIENTATION_TRANSVERSE: *base = (int64_t) (sw - 1) * dw + (sh - 1); *stepX = -dw; *stepY = -1; break;
+ default: *base = 0; *stepX = 1; *stepY = dw; break;
+ }
+}
+
+// Copy each source pixel (whole uint32, so channel order/premult is irrelevant) to its rotated
+// destination, walking TILE x TILE blocks so the scattered writes of a 90/270 transpose stay
+// cache-resident. dst is densely packed (rowBytes == dw*4, no padding), which the affine math relies on.
+static void rotate_tiled(const uint8_t *src, int srcStride, uint32_t *dst,
+ int sw, int sh, int64_t base, int64_t stepX, int64_t stepY) {
+ for (int ty = 0; ty < sh; ty += TILE) {
+ int yEnd = ty + TILE < sh ? ty + TILE : sh;
+ for (int tx = 0; tx < sw; tx += TILE) {
+ int xEnd = tx + TILE < sw ? tx + TILE : sw;
+ for (int sy = ty; sy < yEnd; sy++) {
+ const uint32_t *srcRow = (const uint32_t *) (src + (size_t) sy * srcStride);
+ int64_t idx = base + (int64_t) sy * stepY + (int64_t) tx * stepX;
+ for (int sx = tx; sx < xEnd; sx++) {
+ dst[idx] = srcRow[sx];
+ idx += stepX;
+ }
+ }
+ }
+ }
+}
+
+// Rotates an RGBA_8888 bitmap to the given EXIF orientation into a freshly malloc'd buffer (free it
+// via NativeBuffer.free). Fills outInfo with {width, height, rowBytes} and returns the buffer
+// address, or 0 if the bitmap can't be handled (e.g. a non-8888 format) so the caller can fall back.
+JNIEXPORT jlong JNICALL
+Java_app_alextran_immich_NativeImage_rotate(
+ JNIEnv *env, jclass clazz, jobject bitmap, jint orientation, jintArray outInfo) {
+ AndroidBitmapInfo info;
+ if (AndroidBitmap_getInfo(env, bitmap, &info) != ANDROID_BITMAP_RESULT_SUCCESS) {
+ return 0;
+ }
+ if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888) {
+ return 0;
+ }
+
+ int sw = (int) info.width;
+ int sh = (int) info.height;
+ int dw = swaps_dims(orientation) ? sh : sw;
+ int dh = swaps_dims(orientation) ? sw : sh;
+
+ uint32_t *dst = (uint32_t *) malloc((size_t) dw * dh * 4);
+ if (dst == NULL) {
+ return 0;
+ }
+
+ void *srcPixels = NULL;
+ if (AndroidBitmap_lockPixels(env, bitmap, &srcPixels) != ANDROID_BITMAP_RESULT_SUCCESS) {
+ free(dst);
+ return 0;
+ }
+
+ int64_t base, stepX, stepY;
+ affine_for(orientation, sw, sh, dw, &base, &stepX, &stepY);
+ rotate_tiled((const uint8_t *) srcPixels, (int) info.stride, dst, sw, sh, base, stepX, stepY);
+
+ AndroidBitmap_unlockPixels(env, bitmap);
+
+ jint dims[3] = {dw, dh, dw * 4};
+ (*env)->SetIntArrayRegion(env, outInfo, 0, 3, dims);
+ // Keep ownership in C until the buffer is safely handed back: if outInfo was somehow too small,
+ // SetIntArrayRegion left a pending exception and Kotlin will never receive (or free) dst.
+ if ((*env)->ExceptionCheck(env)) {
+ free(dst);
+ return 0;
+ }
+ return (jlong) dst;
+}
diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/NativeImage.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/NativeImage.kt
new file mode 100644
index 0000000000..b1398d84b8
--- /dev/null
+++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/NativeImage.kt
@@ -0,0 +1,19 @@
+package app.alextran.immich
+
+import android.graphics.Bitmap
+
+object NativeImage {
+ init {
+ // rotate() is compiled into the native_buffer shared lib (which already links jnigraphics).
+ System.loadLibrary("native_buffer")
+ }
+
+ /**
+ * Rotates an RGBA_8888 [bitmap] to the given EXIF [orientation], writing the result into a freshly
+ * malloc'd native buffer. Returns the buffer address (free it with [NativeBuffer.free]) and fills
+ * [outInfo] with {width, height, rowBytes}. Returns 0 when the bitmap can't be handled (e.g. a
+ * non-8888 config) so the caller can fall back.
+ */
+ @JvmStatic
+ external fun rotate(bitmap: Bitmap, orientation: Int, outInfo: IntArray): Long
+}
diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt
index 3babad2e37..6d2499ee0f 100644
--- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt
+++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt
@@ -12,7 +12,9 @@ import android.provider.MediaStore.Images
import android.provider.MediaStore.Video
import android.util.Size
import androidx.annotation.RequiresApi
+import androidx.exifinterface.media.ExifInterface
import app.alextran.immich.NativeBuffer
+import app.alextran.immich.NativeImage
import kotlin.math.*
import java.io.IOException
import java.util.concurrent.Executors
@@ -181,35 +183,88 @@ class LocalImagesImpl(context: Context) : LocalImageApi {
val id = assetId.toLong()
signal.throwIfCanceled()
- val bitmap = if (isVideo) {
- decodeVideoThumbnail(id, size, signal)
- } else {
- decodeImage(id, size, signal)
- }
-
try {
- signal.throwIfCanceled()
- val res = bitmap.toNativeBuffer()
- signal.throwIfCanceled()
+ val res = if (isVideo) {
+ decodeVideoThumbnail(id, size, signal).toNativeBuffer()
+ } else {
+ val (bitmap, orientation) = decodeImage(id, size, signal)
+ signal.throwIfCanceled()
+ if (orientation == ExifInterface.ORIENTATION_NORMAL || orientation == ExifInterface.ORIENTATION_UNDEFINED) {
+ bitmap.toNativeBuffer()
+ } else {
+ rotateToNativeBuffer(bitmap, orientation, signal)
+ }
+ }
+ // Don't re-check cancellation here: res owns a malloc'd buffer, and bailing to CANCELLED would
+ // orphan it. Deliver it; Dart frees the buffer itself if the request was cancelled meanwhile.
callback(Result.success(res))
} catch (e: Exception) {
callback(if (e is OperationCanceledException) CANCELLED else Result.failure(e))
}
}
- private fun decodeImage(id: Long, size: Size, signal: CancellationSignal): Bitmap {
+ // Returns the decoded bitmap plus the EXIF orientation that still needs applying. Only Q+ raw
+ // decodes come back unrotated (ImageDecoder / loadThumbnail skip EXIF for raw like DNG); every
+ // other path already orients itself, so it reports ORIENTATION_NORMAL.
+ private fun decodeImage(id: Long, size: Size, signal: CancellationSignal): Pair<Bitmap, Int> {
signal.throwIfCanceled()
val uri = ContentUris.withAppendedId(Images.Media.EXTERNAL_CONTENT_URI, id)
+ val handleRaw = Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && isRawMime(uri)
+ val orientation = if (handleRaw) rawOrientation(uri) else ExifInterface.ORIENTATION_NORMAL
+
if (size.width <= 0 || size.height <= 0 || size.width > 768 || size.height > 768) {
- return decodeSource(uri, size, signal)
+ // A "load original" request is unsized -> a full-res decode (a sized > 768 just samples to target).
+ return decodeSource(uri, size, signal) to orientation
}
- return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+ val bitmap = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
resolver.loadThumbnail(uri, size, signal)
} else {
signal.setOnCancelListener { Images.Thumbnails.cancelThumbnailRequest(resolver, id) }
Images.Thumbnails.getThumbnail(resolver, id, Images.Thumbnails.MINI_KIND, OPTIONS)
}
+ return bitmap to orientation
+ }
+
+ private fun isRawMime(uri: Uri): Boolean {
+ val mime = resolver.getType(uri) ?: return false
+ return mime.startsWith("image/x-") || mime == "image/dng"
+ }
+
+ private fun rawOrientation(uri: Uri): Int {
+ return resolver.openInputStream(uri)?.use {
+ ExifInterface(it).getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL)
+ } ?: ExifInterface.ORIENTATION_NORMAL
+ }
+
+ // ImageDecoder / loadThumbnail skip EXIF orientation for raw (e.g. DNG) on Q+, so the decoded
+ // bitmap comes back unrotated. Rotate it into the output buffer in native code (one pass, no
+ // intermediate rotated bitmap).
+ private fun rotateToNativeBuffer(bitmap: Bitmap, orientation: Int, signal: CancellationSignal): Map<String, Long> {
+ signal.throwIfCanceled()
+ // Force ARGB_8888: the native rotate needs a lockable 8888 buffer, and toNativeBuffer() below
+ // allocates width*height*4 (an F16/HDR decode would otherwise under-allocate). No-op for the
+ // common already-8888 case.
+ val src = if (bitmap.config != Bitmap.Config.ARGB_8888) {
+ val converted = bitmap.copy(Bitmap.Config.ARGB_8888, false)
+ bitmap.recycle()
+ converted ?: throw IOException("could not convert bitmap to ARGB_8888")
+ } else {
+ bitmap
+ }
+ try {
+ val info = IntArray(3)
+ val pointer = NativeImage.rotate(src, orientation, info)
+ if (pointer == 0L) throw IOException("native rotate failed for orientation $orientation")
+ return mapOf(
+ "pointer" to pointer,
+ "width" to info[0].toLong(),
+ "height" to info[1].toLong(),
+ "rowBytes" to info[2].toLong()
+ )
+ } finally {
+ if (!src.isRecycled) src.recycle()
+ }
}
private fun decodeVideoThumbnail(id: Long, target: Size, signal: CancellationSignal): Bitmap {