High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
fix(mobile): apply exif orientation to android raw photos (#29337)
Santo Shakilaf2efda31057
diff --git a/mobile/android/app/CMakeLists.txt b/mobile/android/app/CMakeLists.txtindex 133bde4fc0..9cad7b915a 100644--- a/mobile/android/app/CMakeLists.txt+++ b/mobile/android/app/CMakeLists.txt@@ -7,6 +7,7 @@ project(native_buffer LANGUAGES C)add_library(native_buffer SHAREDsrc/main/cpp/native_buffer.c+ src/main/cpp/native_image.c)target_link_libraries(native_buffer jnigraphics)diff --git a/mobile/android/app/src/main/cpp/native_image.c b/mobile/android/app/src/main/cpp/native_image.cnew file mode 100644index 0000000000..07bb7e9a2c--- /dev/null+++ b/mobile/android/app/src/main/cpp/native_image.c@@ -0,0 +1,109 @@+#include <jni.h>+#include <stdlib.h>+#include <stdint.h>+#include <android/bitmap.h>++// Cache-friendly block size for the tiled rotation (in pixels). 32x32 uint32 = 4KB, fits L1.+#define TILE 32++// EXIF orientation values (androidx.exifinterface.media.ExifInterface.ORIENTATION_*).+enum {+ ORIENTATION_FLIP_HORIZONTAL = 2,+ ORIENTATION_ROTATE_180 = 3,+ ORIENTATION_FLIP_VERTICAL = 4,+ ORIENTATION_TRANSPOSE = 5,+ ORIENTATION_ROTATE_90 = 6,+ ORIENTATION_TRANSVERSE = 7,+ ORIENTATION_ROTATE_270 = 8,+};++// The orientations that swap width and height. Must stay in sync with affine_for's dim usage.+static int swaps_dims(int o) {+ return o == ORIENTATION_ROTATE_90 || o == ORIENTATION_ROTATE_270 ||+ o == ORIENTATION_TRANSPOSE || o == ORIENTATION_TRANSVERSE;+}++// A source pixel (sx, sy) maps to destination index base + sx*stepX + sy*stepY, where dw is the+// destination width. This affine form covers all 8 EXIF orientations and matches the pixel layout+// of Bitmap.createBitmap(src, matrixForExifOrientation(o)). int64_t so it stays correct on+// armeabi-v7a (32-bit long) regardless of how large MAX_RAW_DECODE_PIXELS grows.+static void affine_for(int o, int sw, int sh, int dw, int64_t *base, int64_t *stepX, int64_t *stepY) {+ switch (o) {+ case ORIENTATION_ROTATE_90: *base = sh - 1; *stepX = dw; *stepY = -1; break;+ case ORIENTATION_ROTATE_270: *base = (int64_t) (sw - 1) * dw; *stepX = -dw; *stepY = 1; break;+ case ORIENTATION_ROTATE_180: *base = (int64_t) (sh - 1) * dw + (sw - 1); *stepX = -1; *stepY = -dw; break;+ case ORIENTATION_FLIP_HORIZONTAL: *base = sw - 1; *stepX = -1; *stepY = dw; break;+ case ORIENTATION_FLIP_VERTICAL: *base = (int64_t) (sh - 1) * dw; *stepX = 1; *stepY = -dw; break;+ case ORIENTATION_TRANSPOSE: *base = 0; *stepX = dw; *stepY = 1; break;+ case ORIENTATION_TRANSVERSE: *base = (int64_t) (sw - 1) * dw + (sh - 1); *stepX = -dw; *stepY = -1; break;+ default: *base = 0; *stepX = 1; *stepY = dw; break;+ }+}++// Copy each source pixel (whole uint32, so channel order/premult is irrelevant) to its rotated+// destination, walking TILE x TILE blocks so the scattered writes of a 90/270 transpose stay+// cache-resident. dst is densely packed (rowBytes == dw*4, no padding), which the affine math relies on.+static void rotate_tiled(const uint8_t *src, int srcStride, uint32_t *dst,+ int sw, int sh, int64_t base, int64_t stepX, int64_t stepY) {+ for (int ty = 0; ty < sh; ty += TILE) {+ int yEnd = ty + TILE < sh ? ty + TILE : sh;+ for (int tx = 0; tx < sw; tx += TILE) {+ int xEnd = tx + TILE < sw ? tx + TILE : sw;+ for (int sy = ty; sy < yEnd; sy++) {+ const uint32_t *srcRow = (const uint32_t *) (src + (size_t) sy * srcStride);+ int64_t idx = base + (int64_t) sy * stepY + (int64_t) tx * stepX;+ for (int sx = tx; sx < xEnd; sx++) {+ dst[idx] = srcRow[sx];+ idx += stepX;+ }+ }+ }+ }+}++// Rotates an RGBA_8888 bitmap to the given EXIF orientation into a freshly malloc'd buffer (free it+// via NativeBuffer.free). Fills outInfo with {width, height, rowBytes} and returns the buffer+// address, or 0 if the bitmap can't be handled (e.g. a non-8888 format) so the caller can fall back.+JNIEXPORT jlong JNICALL+Java_app_alextran_immich_NativeImage_rotate(+ JNIEnv *env, jclass clazz, jobject bitmap, jint orientation, jintArray outInfo) {+ AndroidBitmapInfo info;+ if (AndroidBitmap_getInfo(env, bitmap, &info) != ANDROID_BITMAP_RESULT_SUCCESS) {+ return 0;+ }+ if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888) {+ return 0;+ }++ int sw = (int) info.width;+ int sh = (int) info.height;+ int dw = swaps_dims(orientation) ? sh : sw;+ int dh = swaps_dims(orientation) ? sw : sh;++ uint32_t *dst = (uint32_t *) malloc((size_t) dw * dh * 4);+ if (dst == NULL) {+ return 0;+ }++ void *srcPixels = NULL;+ if (AndroidBitmap_lockPixels(env, bitmap, &srcPixels) != ANDROID_BITMAP_RESULT_SUCCESS) {+ free(dst);+ return 0;+ }++ int64_t base, stepX, stepY;+ affine_for(orientation, sw, sh, dw, &base, &stepX, &stepY);+ rotate_tiled((const uint8_t *) srcPixels, (int) info.stride, dst, sw, sh, base, stepX, stepY);++ AndroidBitmap_unlockPixels(env, bitmap);++ jint dims[3] = {dw, dh, dw * 4};+ (*env)->SetIntArrayRegion(env, outInfo, 0, 3, dims);+ // Keep ownership in C until the buffer is safely handed back: if outInfo was somehow too small,+ // SetIntArrayRegion left a pending exception and Kotlin will never receive (or free) dst.+ if ((*env)->ExceptionCheck(env)) {+ free(dst);+ return 0;+ }+ return (jlong) dst;+}diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/NativeImage.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/NativeImage.ktnew file mode 100644index 0000000000..b1398d84b8--- /dev/null+++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/NativeImage.kt@@ -0,0 +1,19 @@+package app.alextran.immich++import android.graphics.Bitmap++object NativeImage {+ init {+ // rotate() is compiled into the native_buffer shared lib (which already links jnigraphics).+ System.loadLibrary("native_buffer")+ }++ /**+ * Rotates an RGBA_8888 [bitmap] to the given EXIF [orientation], writing the result into a freshly+ * malloc'd native buffer. Returns the buffer address (free it with [NativeBuffer.free]) and fills+ * [outInfo] with {width, height, rowBytes}. Returns 0 when the bitmap can't be handled (e.g. a+ * non-8888 config) so the caller can fall back.+ */+ @JvmStatic+ external fun rotate(bitmap: Bitmap, orientation: Int, outInfo: IntArray): Long+}diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.ktindex 3babad2e37..6d2499ee0f 100644--- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt+++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt@@ -12,7 +12,9 @@ import android.provider.MediaStore.Imagesimport android.provider.MediaStore.Videoimport android.util.Sizeimport androidx.annotation.RequiresApi+import androidx.exifinterface.media.ExifInterfaceimport app.alextran.immich.NativeBuffer+import app.alextran.immich.NativeImageimport kotlin.math.*import java.io.IOExceptionimport java.util.concurrent.Executors@@ -181,35 +183,88 @@ class LocalImagesImpl(context: Context) : LocalImageApi {val id = assetId.toLong()signal.throwIfCanceled()- val bitmap = if (isVideo) {- decodeVideoThumbnail(id, size, signal)- } else {- decodeImage(id, size, signal)- }-try {- signal.throwIfCanceled()- val res = bitmap.toNativeBuffer()- signal.throwIfCanceled()+ val res = if (isVideo) {+ decodeVideoThumbnail(id, size, signal).toNativeBuffer()+ } else {+ val (bitmap, orientation) = decodeImage(id, size, signal)+ signal.throwIfCanceled()+ if (orientation == ExifInterface.ORIENTATION_NORMAL || orientation == ExifInterface.ORIENTATION_UNDEFINED) {+ bitmap.toNativeBuffer()+ } else {+ rotateToNativeBuffer(bitmap, orientation, signal)+ }+ }+ // Don't re-check cancellation here: res owns a malloc'd buffer, and bailing to CANCELLED would+ // orphan it. Deliver it; Dart frees the buffer itself if the request was cancelled meanwhile.callback(Result.success(res))} catch (e: Exception) {callback(if (e is OperationCanceledException) CANCELLED else Result.failure(e))}}- private fun decodeImage(id: Long, size: Size, signal: CancellationSignal): Bitmap {+ // Returns the decoded bitmap plus the EXIF orientation that still needs applying. Only Q+ raw+ // decodes come back unrotated (ImageDecoder / loadThumbnail skip EXIF for raw like DNG); every+ // other path already orients itself, so it reports ORIENTATION_NORMAL.+ private fun decodeImage(id: Long, size: Size, signal: CancellationSignal): Pair<Bitmap, Int> {signal.throwIfCanceled()val uri = ContentUris.withAppendedId(Images.Media.EXTERNAL_CONTENT_URI, id)+ val handleRaw = Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && isRawMime(uri)+ val orientation = if (handleRaw) rawOrientation(uri) else ExifInterface.ORIENTATION_NORMAL+if (size.width <= 0 || size.height <= 0 || size.width > 768 || size.height > 768) {- return decodeSource(uri, size, signal)+ // A "load original" request is unsized -> a full-res decode (a sized > 768 just samples to target).+ return decodeSource(uri, size, signal) to orientation}- return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {+ val bitmap = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {resolver.loadThumbnail(uri, size, signal)} else {signal.setOnCancelListener { Images.Thumbnails.cancelThumbnailRequest(resolver, id) }Images.Thumbnails.getThumbnail(resolver, id, Images.Thumbnails.MINI_KIND, OPTIONS)}+ return bitmap to orientation+ }++ private fun isRawMime(uri: Uri): Boolean {+ val mime = resolver.getType(uri) ?: return false+ return mime.startsWith("image/x-") || mime == "image/dng"+ }++ private fun rawOrientation(uri: Uri): Int {+ return resolver.openInputStream(uri)?.use {+ ExifInterface(it).getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL)+ } ?: ExifInterface.ORIENTATION_NORMAL+ }++ // ImageDecoder / loadThumbnail skip EXIF orientation for raw (e.g. DNG) on Q+, so the decoded+ // bitmap comes back unrotated. Rotate it into the output buffer in native code (one pass, no+ // intermediate rotated bitmap).+ private fun rotateToNativeBuffer(bitmap: Bitmap, orientation: Int, signal: CancellationSignal): Map<String, Long> {+ signal.throwIfCanceled()+ // Force ARGB_8888: the native rotate needs a lockable 8888 buffer, and toNativeBuffer() below+ // allocates width*height*4 (an F16/HDR decode would otherwise under-allocate). No-op for the+ // common already-8888 case.+ val src = if (bitmap.config != Bitmap.Config.ARGB_8888) {+ val converted = bitmap.copy(Bitmap.Config.ARGB_8888, false)+ bitmap.recycle()+ converted ?: throw IOException("could not convert bitmap to ARGB_8888")+ } else {+ bitmap+ }+ try {+ val info = IntArray(3)+ val pointer = NativeImage.rotate(src, orientation, info)+ if (pointer == 0L) throw IOException("native rotate failed for orientation $orientation")+ return mapOf(+ "pointer" to pointer,+ "width" to info[0].toLong(),+ "height" to info[1].toLong(),+ "rowBytes" to info[2].toLong()+ )+ } finally {+ if (!src.isRecycled) src.recycle()+ }}private fun decodeVideoThumbnail(id: Long, target: Size, signal: CancellationSignal): Bitmap {