diff --git a/mobile/android/app/CMakeLists.txt b/mobile/android/app/CMakeLists.txt index 1569f1859e..133bde4fc0 100644 --- a/mobile/android/app/CMakeLists.txt +++ b/mobile/android/app/CMakeLists.txt @@ -8,3 +8,5 @@ project(native_buffer LANGUAGES C) add_library(native_buffer SHARED src/main/cpp/native_buffer.c ) + +target_link_libraries(native_buffer jnigraphics) diff --git a/mobile/android/app/src/main/cpp/native_buffer.c b/mobile/android/app/src/main/cpp/native_buffer.c index 3720d025f6..dfeda4c427 100644 --- a/mobile/android/app/src/main/cpp/native_buffer.c +++ b/mobile/android/app/src/main/cpp/native_buffer.c @@ -1,40 +1,39 @@ #include #include +#include JNIEXPORT jlong JNICALL -Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_allocateNative( - JNIEnv *env, jclass clazz, jint size) { - void *ptr = malloc(size); - return (jlong) ptr; -} - -JNIEXPORT jlong JNICALL -Java_app_alextran_immich_images_ThumbnailsImpl_allocateNative( +Java_app_alextran_immich_images_LocalImagesImpl_allocateNative( JNIEnv *env, jclass clazz, jint size) { void *ptr = malloc(size); return (jlong) ptr; } JNIEXPORT void JNICALL -Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_freeNative( +Java_app_alextran_immich_images_LocalImagesImpl_freeNative( JNIEnv *env, jclass clazz, jlong address) { free((void *) address); } +JNIEXPORT jobject JNICALL +Java_app_alextran_immich_images_LocalImagesImpl_wrapAsBuffer( + JNIEnv *env, jclass clazz, jlong address, jint capacity) { + return (*env)->NewDirectByteBuffer(env, (void *) address, capacity); +} + +JNIEXPORT jlong JNICALL +Java_app_alextran_immich_images_RemoteImagesImpl_lockBitmapPixels( + JNIEnv *env, jclass clazz, jobject bitmap) { + void *pixels = NULL; + int result = AndroidBitmap_lockPixels(env, bitmap, &pixels); + if (result != ANDROID_BITMAP_RESULT_SUCCESS) { + return 0; + } + return (jlong) pixels; +} + JNIEXPORT void JNICALL -Java_app_alextran_immich_images_ThumbnailsImpl_freeNative( - JNIEnv *env, jclass clazz, jlong address) { - free((void *) address); -} - -JNIEXPORT jobject JNICALL -Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_wrapAsBuffer( - JNIEnv *env, jclass clazz, jlong address, jint capacity) { - return (*env)->NewDirectByteBuffer(env, (void *) address, capacity); -} - -JNIEXPORT jobject JNICALL -Java_app_alextran_immich_images_ThumbnailsImpl_wrapAsBuffer( - JNIEnv *env, jclass clazz, jlong address, jint capacity) { - return (*env)->NewDirectByteBuffer(env, (void *) address, capacity); +Java_app_alextran_immich_images_RemoteImagesImpl_unlockBitmapPixels( + JNIEnv *env, jclass clazz, jobject bitmap) { + AndroidBitmap_unlockPixels(env, bitmap); } diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt index cd6474a54e..6c13bc40bb 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt @@ -12,9 +12,6 @@ import android.provider.MediaStore.Images import android.provider.MediaStore.Video import android.util.Size import androidx.annotation.RequiresApi -import app.alextran.immich.images.LocalImagesImpl.Companion.allocateNative -import app.alextran.immich.images.LocalImagesImpl.Companion.freeNative -import app.alextran.immich.images.LocalImagesImpl.Companion.wrapAsBuffer import java.nio.ByteBuffer import kotlin.math.* import java.util.concurrent.Executors @@ -47,9 +44,9 @@ inline fun ImageDecoder.Source.decodeBitmap(target: Size = Size(0, 0)): Bitmap { fun Bitmap.toNativeBuffer(): Map { val size = width * height * 4 - val pointer = allocateNative(size) + val pointer = LocalImagesImpl.allocateNative(size) try { - val buffer = wrapAsBuffer(pointer, size) + val buffer = LocalImagesImpl.wrapAsBuffer(pointer, size) copyPixelsToBuffer(buffer) recycle() return mapOf( @@ -58,7 +55,7 @@ fun Bitmap.toNativeBuffer(): Map { "height" to height.toLong() ) } catch (e: Exception) { - freeNative(pointer) + LocalImagesImpl.freeNative(pointer) recycle() throw e } diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImages.g.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImages.g.kt index d98676ec67..716169ea6f 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImages.g.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImages.g.kt @@ -49,6 +49,7 @@ private open class RemoteImagesPigeonCodec : StandardMessageCodec() { interface RemoteImageApi { fun requestImage(url: String, headers: Map, requestId: Long, callback: (Result>) -> Unit) fun cancelRequest(requestId: Long) + fun releaseImage(requestId: Long) companion object { /** The codec used by RemoteImageApi. */ @@ -99,6 +100,24 @@ interface RemoteImageApi { channel.setMessageHandler(null) } } + run { + val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.immich_mobile.RemoteImageApi.releaseImage$separatedMessageChannelSuffix", codec) + if (api != null) { + channel.setMessageHandler { message, reply -> + val args = message as List + val requestIdArg = args[0] as Long + val wrapped: List = try { + api.releaseImage(requestIdArg) + listOf(null) + } catch (exception: Throwable) { + RemoteImagesPigeonUtils.wrapError(exception) + } + reply.reply(wrapped) + } + } else { + channel.setMessageHandler(null) + } + } } } } diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImagesImpl.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImagesImpl.kt index 69198ad687..e24ca92bb4 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImagesImpl.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImagesImpl.kt @@ -7,7 +7,6 @@ import android.graphics.ColorSpace import android.graphics.ImageDecoder import android.os.Build import android.os.CancellationSignal -import android.util.Size import app.alextran.immich.core.SSLConfig import okhttp3.Call import okhttp3.Callback @@ -31,9 +30,9 @@ data class RemoteRequest( class RemoteImagesImpl(context: Context) : RemoteImageApi { private val requestMap = ConcurrentHashMap() + private val lockedBitmaps = ConcurrentHashMap() init { - System.loadLibrary("native_buffer") cacheDir = context.cacheDir client = buildClient() } @@ -45,17 +44,23 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi { private const val CACHE_SIZE_BYTES = 1024L * 1024 * 1024 val CANCELLED = Result.success>(emptyMap()) - private val decodePool = Executors.newFixedThreadPool( - (Runtime.getRuntime().availableProcessors() / 2).coerceAtLeast(2) - ) + private val decodePool = + Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() / 2 + 1) private var cacheDir: File? = null private var client: OkHttpClient? = null init { + System.loadLibrary("native_buffer") SSLConfig.addListener(::invalidateClient) } + @JvmStatic + external fun lockBitmapPixels(bitmap: Bitmap): Long + + @JvmStatic + external fun unlockBitmapPixels(bitmap: Bitmap) + private fun invalidateClient() { client?.let { it.dispatcher.cancelAll() @@ -123,8 +128,20 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi { signal.throwIfCanceled() val bitmap = decodeImage(bytes) signal.throwIfCanceled() - val res = bitmap.toNativeBuffer() - callback(Result.success(res)) + + val pointer = lockBitmapPixels(bitmap) + if (pointer == 0L) { + bitmap.recycle() + return@execute callback(Result.failure(RuntimeException("Failed to lock bitmap pixels"))) + } + + lockedBitmaps[requestId] = bitmap + callback(Result.success(mapOf( + "pointer" to pointer, + "width" to bitmap.width.toLong(), + "height" to bitmap.height.toLong(), + "rowBytes" to bitmap.rowBytes.toLong() + ))) } catch (e: Exception) { val result = if (signal.isCanceled) CANCELLED else Result.failure(e) callback(result) @@ -138,8 +155,14 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi { } override fun cancelRequest(requestId: Long) { - val request = requestMap.remove(requestId) ?: return - request.cancellationSignal.cancel() + requestMap.remove(requestId)?.cancellationSignal?.cancel() + releaseImage(requestId) + } + + override fun releaseImage(requestId: Long) { + val bitmap = lockedBitmaps.remove(requestId) ?: return + unlockBitmapPixels(bitmap) + bitmap.recycle() } private fun decodeImage(bytes: ByteArray): Bitmap { diff --git a/mobile/ios/Runner/Images/RemoteImages.g.swift b/mobile/ios/Runner/Images/RemoteImages.g.swift index c8af4d8772..a88b13bb42 100644 --- a/mobile/ios/Runner/Images/RemoteImages.g.swift +++ b/mobile/ios/Runner/Images/RemoteImages.g.swift @@ -72,6 +72,7 @@ class RemoteImagesPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable protocol RemoteImageApi { func requestImage(url: String, headers: [String: String], requestId: Int64, completion: @escaping (Result<[String: Int64], Error>) -> Void) func cancelRequest(requestId: Int64) throws + func releaseImage(requestId: Int64) throws } /// Generated setup class from Pigeon to handle messages through the `binaryMessenger`. @@ -114,5 +115,20 @@ class RemoteImageApiSetup { } else { cancelRequestChannel.setMessageHandler(nil) } + let releaseImageChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.RemoteImageApi.releaseImage\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec) + if let api = api { + releaseImageChannel.setMessageHandler { message, reply in + let args = message as! [Any?] + let requestIdArg = args[0] as! Int64 + do { + try api.releaseImage(requestId: requestIdArg) + reply(wrapResult(nil)) + } catch { + reply(wrapError(error)) + } + } + } else { + releaseImageChannel.setMessageHandler(nil) + } } } diff --git a/mobile/ios/Runner/Images/RemoteImagesImpl.swift b/mobile/ios/Runner/Images/RemoteImagesImpl.swift index eee04858e1..5c1e90547b 100644 --- a/mobile/ios/Runner/Images/RemoteImagesImpl.swift +++ b/mobile/ios/Runner/Images/RemoteImagesImpl.swift @@ -1,4 +1,5 @@ -import Accelerate +import CoreImage +import CoreVideo import Flutter import MobileCoreServices import Photos @@ -20,7 +21,7 @@ class RemoteImageApiImpl: NSObject, RemoteImageApi { private static let delegate = RemoteImageApiDelegate() static let session = { let config = URLSessionConfiguration.default - let thumbnailPath = FileManager.default.temporaryDirectory.appendingPathComponent("thumbnails", isDirectory: true) + let thumbnailPath = FileManager.default.temporaryDirectory.appendingPathComponent("thumbnails2", isDirectory: true) try! FileManager.default.createDirectory(at: thumbnailPath, withIntermediateDirectories: true) config.urlCache = URLCache( memoryCapacity: 0, @@ -47,25 +48,20 @@ class RemoteImageApiImpl: NSObject, RemoteImageApi { func cancelRequest(requestId: Int64) { Self.delegate.cancel(requestId: requestId) + Self.delegate.releasePixelBuffer(requestId: requestId) + } + + func releaseImage(requestId: Int64) { + Self.delegate.releasePixelBuffer(requestId: requestId) } } class RemoteImageApiDelegate: NSObject, URLSessionDataDelegate { private static let requestQueue = DispatchQueue(label: "thumbnail.requests", qos: .userInitiated) - private static var rgbaFormat = vImage_CGImageFormat( - bitsPerComponent: 8, - bitsPerPixel: 32, - colorSpace: CGColorSpaceCreateDeviceRGB(), - bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue), - renderingIntent: .perceptual - )! private static var requests = [Int64: RemoteImageRequest]() + private static var lockedPixelBuffers = [Int64: CVPixelBuffer]() private static let cancelledResult = Result<[String: Int64], any Error>.success([:]) - private static let decodeOptions = [ - kCGImageSourceShouldCache: false, - kCGImageSourceShouldCacheImmediately: true, - kCGImageSourceCreateThumbnailWithTransform: true, - ] as CFDictionary + private static let ciContext = CIContext(options: [.useSoftwareRenderer: false]) func urlSession( _ session: URLSession, dataTask: URLSessionDataTask, @@ -108,11 +104,13 @@ class RemoteImageApiDelegate: NSObject, URLSessionDataDelegate { defer { remove(requestId: requestId) } if let error = error { + if request.isCancelled || (error as NSError).code == NSURLErrorCancelled { + return request.completion(Self.cancelledResult) + } return request.completion(.failure(error)) } - guard let imageSource = CGImageSourceCreateWithData(data, nil), - let cgImage = CGImageSourceCreateThumbnailAtIndex(imageSource, 0, Self.decodeOptions) else { + guard let ciImage = CIImage(data: data as Data, options: [.applyOrientationProperty: true]) else { return request.completion(.failure(PigeonError(code: "", message: "Failed to decode image for request \(requestId)", details: nil))) } @@ -120,24 +118,52 @@ class RemoteImageApiDelegate: NSObject, URLSessionDataDelegate { return request.completion(Self.cancelledResult) } - do { - let buffer = try vImage_Buffer(cgImage: cgImage, format: Self.rgbaFormat) - - if request.isCancelled { - buffer.free() - return request.completion(Self.cancelledResult) - } - - request.completion( - .success([ - "pointer": Int64(Int(bitPattern: buffer.data)), - "width": Int64(buffer.width), - "height": Int64(buffer.height), - "rowBytes": Int64(buffer.rowBytes), - ])) - } catch { - return request.completion(.failure(PigeonError(code: "", message: "Failed to convert image for request \(requestId): \(error)", details: nil))) + let extent = ciImage.extent + let width = Int(extent.width) + let height = Int(extent.height) + + guard width > 0 && height > 0 else { + return request.completion(.failure(PigeonError(code: "", message: "Invalid image dimensions \(width)x\(height) for request \(requestId)", details: nil))) } + + var pixelBuffer: CVPixelBuffer? + let attrs: [String: Any] = [ + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, + kCVPixelBufferWidthKey as String: width, + kCVPixelBufferHeightKey as String: height, + kCVPixelBufferIOSurfacePropertiesKey as String: [:], + ] + let status = CVPixelBufferCreate(kCFAllocatorDefault, width, height, kCVPixelFormatType_32BGRA, attrs as CFDictionary, &pixelBuffer) + + guard status == kCVReturnSuccess, let pixelBuffer = pixelBuffer else { + return request.completion(.failure(PigeonError(code: "", message: "Failed to create pixel buffer for request \(requestId), status: \(status)", details: nil))) + } + + if request.isCancelled { + return request.completion(Self.cancelledResult) + } + + Self.ciContext.render(ciImage, to: pixelBuffer) + + CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly) + guard let pointer = CVPixelBufferGetBaseAddress(pixelBuffer) else { + CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly) + return request.completion(.failure(PigeonError(code: "", message: "Failed to lock pixel buffer for request \(requestId)", details: nil))) + } + + let rowBytes = CVPixelBufferGetBytesPerRow(pixelBuffer) + + Self.requestQueue.sync { + Self.lockedPixelBuffers[requestId] = pixelBuffer + } + + request.completion( + .success([ + "pointer": Int64(Int(bitPattern: pointer)), + "width": Int64(width), + "height": Int64(height), + "rowBytes": Int64(rowBytes), + ])) } func get(requestId: Int64) -> RemoteImageRequest? { @@ -157,4 +183,9 @@ class RemoteImageApiDelegate: NSObject, URLSessionDataDelegate { request.isCancelled = true request.task?.cancel() } + + func releasePixelBuffer(requestId: Int64) -> Void { + guard let pixelBuffer = (Self.requestQueue.sync { Self.lockedPixelBuffers.removeValue(forKey: requestId) }) else { return } + CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly) + } } diff --git a/mobile/lib/infrastructure/loaders/image_request.dart b/mobile/lib/infrastructure/loaders/image_request.dart index 424a6f350b..8aed6113bb 100644 --- a/mobile/lib/infrastructure/loaders/image_request.dart +++ b/mobile/lib/infrastructure/loaders/image_request.dart @@ -34,7 +34,7 @@ abstract class ImageRequest { void _onCancelled(); - Future _fromPlatformImage(Map info) async { + Future _fromPlatformImage(Map info, ui.PixelFormat pixelFormat, bool shouldFree) async { final address = info['pointer']; if (address == null) { return null; @@ -42,7 +42,9 @@ abstract class ImageRequest { final pointer = Pointer.fromAddress(address); if (_isCancelled) { - malloc.free(pointer); + if (shouldFree) { + malloc.free(pointer); + } return null; } @@ -58,7 +60,9 @@ abstract class ImageRequest { actualSize = rowBytes * actualHeight; buffer = await ImmutableBuffer.fromUint8List(pointer.asTypedList(actualSize)); } finally { - malloc.free(pointer); + if (shouldFree) { + malloc.free(pointer); + } } if (_isCancelled) { @@ -71,7 +75,7 @@ abstract class ImageRequest { width: actualWidth, height: actualHeight, rowBytes: rowBytes, - pixelFormat: ui.PixelFormat.rgba8888, + pixelFormat: pixelFormat, ); final codec = await descriptor.instantiateCodec(); if (_isCancelled) { diff --git a/mobile/lib/infrastructure/loaders/local_image_request.dart b/mobile/lib/infrastructure/loaders/local_image_request.dart index ace83a8709..d78afdd151 100644 --- a/mobile/lib/infrastructure/loaders/local_image_request.dart +++ b/mobile/lib/infrastructure/loaders/local_image_request.dart @@ -24,7 +24,7 @@ class LocalImageRequest extends ImageRequest { isVideo: assetType == AssetType.video, ); - final frame = await _fromPlatformImage(info); + final frame = await _fromPlatformImage(info, ui.PixelFormat.rgba8888, true); return frame == null ? null : ImageInfo(image: frame.image, scale: scale); } diff --git a/mobile/lib/infrastructure/loaders/remote_image_request.dart b/mobile/lib/infrastructure/loaders/remote_image_request.dart index b3f4ec226c..1d53d46543 100644 --- a/mobile/lib/infrastructure/loaders/remote_image_request.dart +++ b/mobile/lib/infrastructure/loaders/remote_image_request.dart @@ -14,12 +14,16 @@ class RemoteImageRequest extends ImageRequest { final Map info = await remoteImageApi.requestImage(uri, headers: headers, requestId: requestId); - final frame = await _fromPlatformImage(info); - return frame == null ? null : ImageInfo(image: frame.image, scale: scale); + try { + final frame = await _fromPlatformImage(info, ui.PixelFormat.bgra8888, false); + return frame == null ? null : ImageInfo(image: frame.image, scale: scale); + } finally { + unawaited(remoteImageApi.releaseImage(requestId)); + } } @override Future _onCancelled() { - return localImageApi.cancelRequest(requestId); + return remoteImageApi.cancelRequest(requestId); } } diff --git a/mobile/lib/infrastructure/loaders/thumbhash_image_request.dart b/mobile/lib/infrastructure/loaders/thumbhash_image_request.dart index 1a184d1cb8..a33f8c51c4 100644 --- a/mobile/lib/infrastructure/loaders/thumbhash_image_request.dart +++ b/mobile/lib/infrastructure/loaders/thumbhash_image_request.dart @@ -12,7 +12,7 @@ class ThumbhashImageRequest extends ImageRequest { } final Map info = await localImageApi.getThumbhash(thumbhash); - final frame = await _fromPlatformImage(info); + final frame = await _fromPlatformImage(info, ui.PixelFormat.rgba8888, true); return frame == null ? null : ImageInfo(image: frame.image, scale: scale); } diff --git a/mobile/lib/platform/remote_image_api.g.dart b/mobile/lib/platform/remote_image_api.g.dart index f9788ea979..63970979da 100644 --- a/mobile/lib/platform/remote_image_api.g.dart +++ b/mobile/lib/platform/remote_image_api.g.dart @@ -103,4 +103,27 @@ class RemoteImageApi { return; } } + + Future releaseImage(int requestId) async { + final String pigeonVar_channelName = + 'dev.flutter.pigeon.immich_mobile.RemoteImageApi.releaseImage$pigeonVar_messageChannelSuffix'; + final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( + pigeonVar_channelName, + pigeonChannelCodec, + binaryMessenger: pigeonVar_binaryMessenger, + ); + final Future pigeonVar_sendFuture = pigeonVar_channel.send([requestId]); + final List? pigeonVar_replyList = await pigeonVar_sendFuture as List?; + if (pigeonVar_replyList == null) { + throw _createConnectionError(pigeonVar_channelName); + } else if (pigeonVar_replyList.length > 1) { + throw PlatformException( + code: pigeonVar_replyList[0]! as String, + message: pigeonVar_replyList[1] as String?, + details: pigeonVar_replyList[2], + ); + } else { + return; + } + } } diff --git a/mobile/pigeon/remote_image_api.dart b/mobile/pigeon/remote_image_api.dart index 5ff73631f9..e43013ac6a 100644 --- a/mobile/pigeon/remote_image_api.dart +++ b/mobile/pigeon/remote_image_api.dart @@ -22,4 +22,6 @@ abstract class RemoteImageApi { }); void cancelRequest(int requestId); + + void releaseImage(int requestId); }