π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[mob][photos] Add ente_vision plugin for Apple Vision OCR
laurenspriem2fbbcbc6517c
diff --git a/mobile/apps/photos/ios/Podfile.lock b/mobile/apps/photos/ios/Podfile.lockindex e5ddacb279..253254105d 100644--- a/mobile/apps/photos/ios/Podfile.lock+++ b/mobile/apps/photos/ios/Podfile.lock@@ -30,6 +30,8 @@ PODS:- Flutter- ente_screen_cover (0.0.1):- Flutter+ - ente_vision (0.0.1):+ - Flutter- EnteOnnxRuntime (1.28.1-r1)- ffmpeg_kit_custom (6.0.3)- ffmpeg_kit_flutter (6.0.3):@@ -255,6 +257,7 @@ DEPENDENCIES:- ente_qr (from `.symlinks/plugins/ente_qr/ios`)- ente_screen_brightness (from `.symlinks/plugins/ente_screen_brightness/ios`)- ente_screen_cover (from `.symlinks/plugins/ente_screen_cover/ios`)+ - ente_vision (from `.symlinks/plugins/ente_vision/ios`)- EnteOnnxRuntime (from `../../../../apple/packages/EnteOnnxRuntime/EnteOnnxRuntime.podspec`)- ffmpeg_kit_flutter (from `.symlinks/plugins/ffmpeg_kit_flutter/ios`)- file_saver (from `.symlinks/plugins/file_saver/ios`)@@ -350,6 +353,8 @@ EXTERNAL SOURCES::path: ".symlinks/plugins/ente_screen_brightness/ios"ente_screen_cover::path: ".symlinks/plugins/ente_screen_cover/ios"+ ente_vision:+ :path: ".symlinks/plugins/ente_vision/ios"EnteOnnxRuntime::podspec: "../../../../apple/packages/EnteOnnxRuntime/EnteOnnxRuntime.podspec"ffmpeg_kit_flutter:@@ -451,6 +456,7 @@ SPEC CHECKSUMS:ente_qr: 86beaf5d2644705db9b565b32dd8b82c05fb36afente_screen_brightness: 7dd13c361508ee6098a2578908ebf35e9f567e28ente_screen_cover: 6792566774f1c83939f9c1d03ed0a1f1ded45ae4+ ente_vision: 260965fb4a3bda0c0cb72c5f15eb8748fef7c897EnteOnnxRuntime: 7baa83a95717a6c9c8d0d4599f9f2c90b72d2905ffmpeg_kit_custom: 682b4f2f1ff1f8abae5a92f6c3540f2441d5be99ffmpeg_kit_flutter: 915b345acc97d4142e8a9a8549d177ff10f043f5diff --git a/mobile/apps/photos/ios/Runner.xcodeproj/project.pbxproj b/mobile/apps/photos/ios/Runner.xcodeproj/project.pbxprojindex 1f5453d1d3..ba96b8e348 100644--- a/mobile/apps/photos/ios/Runner.xcodeproj/project.pbxproj+++ b/mobile/apps/photos/ios/Runner.xcodeproj/project.pbxproj@@ -645,6 +645,7 @@"${BUILT_PRODUCTS_DIR}/ente_qr/ente_qr.framework","${BUILT_PRODUCTS_DIR}/ente_screen_brightness/ente_screen_brightness.framework","${BUILT_PRODUCTS_DIR}/ente_screen_cover/ente_screen_cover.framework",+ "${BUILT_PRODUCTS_DIR}/ente_vision/ente_vision.framework","${BUILT_PRODUCTS_DIR}/file_saver/file_saver.framework","${BUILT_PRODUCTS_DIR}/flutter_image_compress_common/flutter_image_compress_common.framework","${BUILT_PRODUCTS_DIR}/flutter_inappwebview_ios/flutter_inappwebview_ios.framework",@@ -661,7 +662,6 @@"${BUILT_PRODUCTS_DIR}/launcher_icon_switcher/launcher_icon_switcher.framework","${BUILT_PRODUCTS_DIR}/libwebp/libwebp.framework","${BUILT_PRODUCTS_DIR}/local_auth_darwin/local_auth_darwin.framework",- "${BUILT_PRODUCTS_DIR}/media_extension/media_extension.framework","${BUILT_PRODUCTS_DIR}/media_kit_libs_ios_video/media_kit_libs_ios_video.framework","${BUILT_PRODUCTS_DIR}/media_kit_video/media_kit_video.framework","${BUILT_PRODUCTS_DIR}/mobile_ocr/mobile_ocr.framework",@@ -741,6 +741,7 @@"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/ente_qr.framework","${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/ente_screen_brightness.framework","${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/ente_screen_cover.framework",+ "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/ente_vision.framework","${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/file_saver.framework","${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/flutter_image_compress_common.framework","${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/flutter_inappwebview_ios.framework",@@ -757,7 +758,6 @@"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/launcher_icon_switcher.framework","${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/libwebp.framework","${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/local_auth_darwin.framework",- "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/media_extension.framework","${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/media_kit_libs_ios_video.framework","${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/media_kit_video.framework","${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/mobile_ocr.framework",diff --git a/mobile/apps/photos/plugins/ente_vision/ios/.swift-format b/mobile/apps/photos/plugins/ente_vision/ios/.swift-formatnew file mode 100644index 0000000000..46b20d5169--- /dev/null+++ b/mobile/apps/photos/plugins/ente_vision/ios/.swift-format@@ -0,0 +1,13 @@+{+ "indentation": { "spaces": 4 },+ "lineLength": 120,+ "respectsExistingLineBreaks": true,+ "rules": {+ "AllPublicDeclarationsHaveDocumentation": false,+ "AlwaysUseLowerCamelCase": true,+ "DoNotUseSemicolons": true,+ "FileScopedDeclarationPrivacy": true,+ "NeverForceUnwrap": true,+ "NeverUseImplicitlyUnwrappedOptionals": true+ }+}diff --git a/mobile/apps/photos/plugins/ente_vision/ios/Flutter/EnteVisionPlugin.swift b/mobile/apps/photos/plugins/ente_vision/ios/Flutter/EnteVisionPlugin.swiftnew file mode 100644index 0000000000..9edec741b2--- /dev/null+++ b/mobile/apps/photos/plugins/ente_vision/ios/Flutter/EnteVisionPlugin.swift@@ -0,0 +1,20 @@+@preconcurrency import Flutter+import Foundation++@MainActor+public final class EnteVisionPlugin: NSObject, @preconcurrency FlutterPlugin {+ private let textRecognitionAdapter: TextRecognitionChannelAdapter++ private init(registrar: FlutterPluginRegistrar) {+ textRecognitionAdapter = TextRecognitionChannelAdapter(registrar: registrar)+ super.init()+ }++ public static func register(with registrar: FlutterPluginRegistrar) {+ registrar.publish(EnteVisionPlugin(registrar: registrar))+ }++ public func detachFromEngine(for registrar: FlutterPluginRegistrar) {+ textRecognitionAdapter.detach()+ }+}diff --git a/mobile/apps/photos/plugins/ente_vision/ios/Flutter/TextRecognitionChannelAdapter.swift b/mobile/apps/photos/plugins/ente_vision/ios/Flutter/TextRecognitionChannelAdapter.swiftnew file mode 100644index 0000000000..76d2a1c6d7--- /dev/null+++ b/mobile/apps/photos/plugins/ente_vision/ios/Flutter/TextRecognitionChannelAdapter.swift@@ -0,0 +1,189 @@+@preconcurrency import Flutter+import Foundation++@MainActor+final class TextRecognitionChannelAdapter {+ private let channel: FlutterMethodChannel+ private let recognizer = TextRecognizer()+ private var isAttached = true++ init(registrar: FlutterPluginRegistrar) {+ channel = FlutterMethodChannel(+ name: Self.channelName,+ binaryMessenger: registrar.messenger()+ )+ channel.setMethodCallHandler { [weak self] call, result in+ self?.handle(call, result: result)+ }+ }++ func detach() {+ guard isAttached else { return }+ isAttached = false+ recognizer.cancelAll()+ channel.setMethodCallHandler(nil)+ }++ private func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) {+ guard isAttached else {+ result(TextRecognitionError.cancelled.flutterError)+ return+ }+ switch call.method {+ case "textRecognition.detectText":+ handleDetectText(call, result: result)+ case "textRecognition.detectTextRegions":+ handleDetectTextRegions(call, result: result)+ case "textRecognition.cancelRequest":+ handleCancelRequest(call, result: result)+ default:+ result(FlutterMethodNotImplemented)+ }+ }++ private func handleDetectText(_ call: FlutterMethodCall, result: @escaping FlutterResult) {+ guard let arguments = call.arguments as? [String: Any],+ let imagePath = arguments["imagePath"] as? String+ else {+ result(invalidArgument("imagePath is required"))+ return+ }+ recognizer.recognizeText(+ imagePath: imagePath,+ includeAllConfidenceScores: arguments["includeAllConfidenceScores"] as? Bool ?? false,+ requestId: arguments["requestId"] as? String+ ) { outcome in+ deliver(outcome, to: result, channelValue: \.channelValue)+ }+ }++ private func handleDetectTextRegions(_ call: FlutterMethodCall, result: @escaping FlutterResult) {+ guard let arguments = call.arguments as? [String: Any],+ let imagePath = arguments["imagePath"] as? String+ else {+ result(invalidArgument("imagePath is required"))+ return+ }+ recognizer.detectTextRegions(+ imagePath: imagePath,+ requestId: arguments["requestId"] as? String+ ) { outcome in+ deliver(outcome, to: result, channelValue: \.channelValue)+ }+ }++ private func handleCancelRequest(_ call: FlutterMethodCall, result: FlutterResult) {+ guard let arguments = call.arguments as? [String: Any],+ let requestId = arguments["requestId"] as? String, !requestId.isEmpty+ else {+ result(invalidArgument("requestId is required"))+ return+ }+ recognizer.cancel(requestId: requestId)+ result(nil)+ }++ private static let channelName = "io.ente.photos.vision/text_recognition"+}++private func invalidArgument(_ message: String) -> FlutterError {+ FlutterError(code: "INVALID_ARGUMENT", message: message, details: nil)+}++private func deliver<Value: Sendable>(+ _ outcome: Result<Value, TextRecognitionError>,+ to result: @escaping FlutterResult,+ channelValue: @escaping @Sendable (Value) -> [String: Any]+) {+ DispatchQueue.main.async {+ switch outcome {+ case .success(let value):+ result(channelValue(value))+ case .failure(let error):+ result(error.flutterError)+ }+ }+}++extension TextRecognitionError {+ fileprivate var flutterError: FlutterError {+ switch self {+ case .imageNotFound:+ FlutterError(code: "IMAGE_NOT_FOUND", message: "Image file does not exist", details: nil)+ case .imageUndecodable:+ FlutterError(code: "IMAGE_DECODE_ERROR", message: "Failed to load image from path", details: nil)+ case .imageBitmapUnavailable:+ FlutterError(code: "IMAGE_DECODE_ERROR", message: "Failed to get CGImage", details: nil)+ case .cancelled:+ FlutterError(code: "CANCELLED", message: "OCR request was cancelled", details: nil)+ case .detectionFailed(let error):+ FlutterError(+ code: "DETECTION_ERROR",+ message: "Failed to detect text regions",+ details: error.localizedDescription+ )+ case .recognitionFailed(let error):+ FlutterError(+ code: "RECOGNITION_ERROR",+ message: "Text recognition failed",+ details: error.localizedDescription+ )+ }+ }+}++extension TextRecognitionResult {+ fileprivate var channelValue: [String: Any] {+ [+ "blocks": blocks.map(\.channelValue),+ "imageWidth": imageWidth,+ "imageHeight": imageHeight,+ ]+ }+}++extension RecognizedTextBlock {+ fileprivate var channelValue: [String: Any] {+ [+ "text": text,+ "confidence": Double(confidence),+ "points": points.channelValue,+ "characters": characters.map(\.channelValue),+ ]+ }+}++extension RecognizedCharacter {+ fileprivate var channelValue: [String: Any] {+ [+ "text": text,+ "confidence": Double(confidence),+ "points": points.channelValue,+ ]+ }+}++extension TextRegionsResult {+ fileprivate var channelValue: [String: Any] {+ [+ "regions": regions.map(\.channelValue),+ "imageWidth": imageWidth,+ "imageHeight": imageHeight,+ ]+ }+}++extension TextRegion {+ fileprivate var channelValue: [String: Any] {+ [+ "confidence": Double(confidence),+ "points": points.channelValue,+ ]+ }+}++extension [CGPoint] {+ fileprivate var channelValue: [[String: Double]] {+ map { ["x": Double($0.x), "y": Double($0.y)] }+ }+}diff --git a/mobile/apps/photos/plugins/ente_vision/ios/Sources/EnteVisionCore/RecognitionLanguageSelector.swift b/mobile/apps/photos/plugins/ente_vision/ios/Sources/EnteVisionCore/RecognitionLanguageSelector.swiftnew file mode 100644index 0000000000..11804a1853--- /dev/null+++ b/mobile/apps/photos/plugins/ente_vision/ios/Sources/EnteVisionCore/RecognitionLanguageSelector.swift@@ -0,0 +1,112 @@+import Foundation++enum RecognitionLanguageSelector {+ static func select(+ preferredLanguages: [String],+ supportedLanguages: [String],+ limit: Int = 3+ ) -> [String] {+ guard limit > 0, !supportedLanguages.isEmpty else { return [] }++ var selected: [String] = []+ for preferred in preferredLanguages {+ let remaining = supportedLanguages.filter { !selected.contains($0) }+ if let match = bestMatch(for: preferred, in: remaining) {+ selected.append(match)+ }+ if selected.count == limit {+ return selected+ }+ }++ if selected.count < limit,+ let english = supportedLanguages.first(where: {+ normalized($0) == "en-us"+ }),+ !selected.contains(english)+ {+ selected.append(english)+ }++ return selected.isEmpty+ ? Array(supportedLanguages.prefix(1))+ : selected+ }++ private static func bestMatch(+ for preferred: String,+ in supportedLanguages: [String]+ ) -> String? {+ if let exact = supportedLanguages.first(where: {+ normalized($0) == normalized(preferred)+ }) {+ return exact+ }++ let preferredTag = LanguageTag(preferred)+ guard !preferredTag.language.isEmpty else { return nil }+ let supportedTags = supportedLanguages.map {+ (identifier: $0, tag: LanguageTag($0))+ }++ if let script = preferredTag.effectiveScript,+ let scriptMatch = supportedTags.first(where: {+ $0.tag.language == preferredTag.language && $0.tag.effectiveScript == script+ })+ {+ return scriptMatch.identifier+ }++ if let region = preferredTag.region,+ let regionMatch = supportedTags.first(where: {+ $0.tag.language == preferredTag.language && $0.tag.region == region+ })+ {+ return regionMatch.identifier+ }++ return supportedTags.first(where: {+ $0.tag.language == preferredTag.language+ })?.identifier+ }++ private static func normalized(_ identifier: String) -> String {+ identifier.replacingOccurrences(of: "_", with: "-").lowercased()+ }+}++private struct LanguageTag {+ let language: String+ let script: String?+ let region: String?++ init(_ identifier: String) {+ let parts =+ identifier+ .replacingOccurrences(of: "_", with: "-")+ .split(separator: "-")+ .map(String.init)+ language = parts.first?.lowercased() ?? ""+ script = parts.dropFirst().first(where: { part in+ part.count == 4 && part.allSatisfy(\.isLetter)+ })?.lowercased()+ region = parts.dropFirst().first(where: { part in+ (part.count == 2 && part.allSatisfy(\.isLetter)) || (part.count == 3 && part.allSatisfy(\.isNumber))+ })?.uppercased()+ }++ var effectiveScript: String? {+ if let script = script {+ return script+ }+ guard language == "zh", let region = region else { return nil }+ switch region {+ case "HK", "MO", "TW":+ return "hant"+ case "CN", "MY", "SG":+ return "hans"+ default:+ return nil+ }+ }+}diff --git a/mobile/apps/photos/plugins/ente_vision/ios/Sources/EnteVisionCore/TextRecognizer.swift b/mobile/apps/photos/plugins/ente_vision/ios/Sources/EnteVisionCore/TextRecognizer.swiftnew file mode 100644index 0000000000..75db29e61c--- /dev/null+++ b/mobile/apps/photos/plugins/ente_vision/ios/Sources/EnteVisionCore/TextRecognizer.swift@@ -0,0 +1,318 @@+import Foundation+import UIKit+import Vision++struct RecognizedCharacter: Sendable {+ let text: String+ let confidence: Float+ let points: [CGPoint]+}++struct RecognizedTextBlock: Sendable {+ let text: String+ let confidence: Float+ let points: [CGPoint]+ let characters: [RecognizedCharacter]+}++struct TextRecognitionResult: Sendable {+ let blocks: [RecognizedTextBlock]+ let imageWidth: Int+ let imageHeight: Int+}++struct TextRegion: Sendable {+ let confidence: Float+ let points: [CGPoint]+}++struct TextRegionsResult: Sendable {+ let regions: [TextRegion]+ let imageWidth: Int+ let imageHeight: Int+}++enum TextRecognitionError: Error {+ case imageNotFound+ case imageUndecodable+ case imageBitmapUnavailable+ case cancelled+ case detectionFailed(Error)+ case recognitionFailed(Error)+}++final class TextRecognizer: @unchecked Sendable {+ private let lock = NSLock()+ private var activeRequests: [RequestKey: RequestState] = [:]+ private let queue = DispatchQueue.global(qos: .userInitiated)++ func recognizeText(+ imagePath: String,+ includeAllConfidenceScores: Bool,+ requestId: String?,+ completion: @escaping @Sendable (Result<TextRecognitionResult, TextRecognitionError>) -> Void+ ) {+ let minimumConfidence: Float = includeAllConfidenceScores ? 0 : Self.minimumConfidence+ run(requestId: requestId, foreignError: { .recognitionFailed($0) }, completion: completion) {+ [self] state in+ let image = try loadOrientedImage(atPath: imagePath, state: state)+ let request = Self.makeRecognizeTextRequest()+ try attach(request, to: state)+ try VNImageRequestHandler(cgImage: image.bitmap, options: [:]).perform([request])+ let size = image.pixelSize+ let blocks = (request.results ?? []).compactMap { observation -> RecognizedTextBlock? in+ guard let candidate = observation.topCandidates(1).first,+ candidate.confidence >= minimumConfidence+ else {+ return nil+ }+ return RecognizedTextBlock(+ text: candidate.string,+ confidence: candidate.confidence,+ points: observation.polygon(in: size),+ characters: Self.characters(of: candidate, in: size)+ )+ }+ return TextRecognitionResult(+ blocks: blocks.sortedInReadingOrder(),+ imageWidth: image.bitmap.width,+ imageHeight: image.bitmap.height+ )+ }+ }++ func detectTextRegions(+ imagePath: String,+ requestId: String?,+ completion: @escaping @Sendable (Result<TextRegionsResult, TextRecognitionError>) -> Void+ ) {+ run(requestId: requestId, foreignError: { .detectionFailed($0) }, completion: completion) {+ [self] state in+ let image = try loadOrientedImage(atPath: imagePath, state: state)+ let detectionBitmap =+ Self.downscaled(image.image, longestSide: Self.regionDetectionLongestSide) ?? image.bitmap+ let request = VNDetectTextRectanglesRequest()+ request.reportCharacterBoxes = false+ try attach(request, to: state)+ try VNImageRequestHandler(cgImage: detectionBitmap, options: [:]).perform([request])+ let size = image.pixelSize+ let regions = (request.results ?? []).map { observation in+ TextRegion(confidence: observation.confidence, points: observation.polygon(in: size))+ }+ return TextRegionsResult(+ regions: regions,+ imageWidth: image.bitmap.width,+ imageHeight: image.bitmap.height+ )+ }+ }++ func cancel(requestId: String) {+ lock.lock()+ let state = activeRequests.removeValue(forKey: .client(requestId))+ state?.isCancelled = true+ let request = state?.request+ lock.unlock()+ request?.cancel()+ }++ func cancelAll() {+ lock.lock()+ let states = Array(activeRequests.values)+ activeRequests.removeAll()+ for state in states {+ state.isCancelled = true+ }+ let requests = states.compactMap(\.request)+ lock.unlock()+ for request in requests {+ request.cancel()+ }+ }++ private func run<Value>(+ requestId: String?,+ foreignError: @escaping @Sendable (Error) -> TextRecognitionError,+ completion: @escaping @Sendable (Result<Value, TextRecognitionError>) -> Void,+ _ work: @escaping @Sendable (RequestState) throws -> Value+ ) {+ let state = begin(requestId: requestId)+ queue.async {+ let outcome = Result { try work(state) }+ .mapError { $0 as? TextRecognitionError ?? foreignError($0) }+ completion(self.finish(state, outcome))+ }+ }++ private func begin(requestId: String?) -> RequestState {+ let state = RequestState(key: requestId.map(RequestKey.client) ?? .anonymous(UUID()))+ lock.lock()+ let previous = activeRequests.updateValue(state, forKey: state.key)+ previous?.isCancelled = true+ let previousRequest = previous?.request+ lock.unlock()+ previousRequest?.cancel()+ return state+ }++ private func attach(_ request: VNRequest, to state: RequestState) throws {+ lock.lock()+ defer { lock.unlock() }+ guard !state.isCancelled else { throw TextRecognitionError.cancelled }+ state.request = request+ }++ private func checkNotCancelled(_ state: RequestState) throws {+ lock.lock()+ defer { lock.unlock() }+ if state.isCancelled {+ throw TextRecognitionError.cancelled+ }+ }++ private func finish<Value>(+ _ state: RequestState,+ _ outcome: Result<Value, TextRecognitionError>+ ) -> Result<Value, TextRecognitionError> {+ lock.lock()+ defer { lock.unlock() }+ if activeRequests[state.key] === state {+ activeRequests.removeValue(forKey: state.key)+ }+ state.request = nil+ return state.isCancelled ? .failure(.cancelled) : outcome+ }++ private func loadOrientedImage(atPath path: String, state: RequestState) throws -> OrientedImage {+ try checkNotCancelled(state)+ guard FileManager.default.fileExists(atPath: path) else { throw TextRecognitionError.imageNotFound }+ guard let image = UIImage(contentsOfFile: path) else { throw TextRecognitionError.imageUndecodable }+ try checkNotCancelled(state)+ let oriented = Self.orientedUp(image)+ guard let bitmap = oriented.cgImage else { throw TextRecognitionError.imageBitmapUnavailable }+ try checkNotCancelled(state)+ return OrientedImage(image: oriented, bitmap: bitmap)+ }++ private static func makeRecognizeTextRequest() -> VNRecognizeTextRequest {+ let request = VNRecognizeTextRequest()+ request.recognitionLevel = .accurate+ request.minimumTextHeight = 0.01+ request.usesLanguageCorrection = true+ if #available(iOS 16.0, *) {+ request.automaticallyDetectsLanguage = true+ request.revision = VNRecognizeTextRequestRevision3+ } else {+ request.recognitionLanguages = preferredRecognitionLanguages(for: request)+ }+ return request+ }++ private static func preferredRecognitionLanguages(for request: VNRecognizeTextRequest) -> [String] {+ let supported = (try? request.supportedRecognitionLanguages()) ?? ["en-US"]+ return RecognitionLanguageSelector.select(+ preferredLanguages: Locale.preferredLanguages,+ supportedLanguages: supported+ )+ }++ private static func characters(of candidate: VNRecognizedText, in size: CGSize) -> [RecognizedCharacter] {+ let text = candidate.string+ return text.indices.compactMap { start in+ let range = start..<text.index(after: start)+ guard let box = try? candidate.boundingBox(for: range) else { return nil }+ return RecognizedCharacter(+ text: String(text[range]),+ confidence: candidate.confidence,+ points: box.polygon(in: size)+ )+ }+ }++ private static func orientedUp(_ image: UIImage) -> UIImage {+ guard image.imageOrientation != .up else { return image }+ let format = UIGraphicsImageRendererFormat.default()+ format.scale = image.scale+ format.opaque = false+ return UIGraphicsImageRenderer(size: image.size, format: format).image { _ in+ image.draw(in: CGRect(origin: .zero, size: image.size))+ }+ }++ private static func downscaled(_ image: UIImage, longestSide maxDimension: CGFloat) -> CGImage? {+ guard let source = image.cgImage else { return nil }+ let width = CGFloat(source.width)+ let height = CGFloat(source.height)+ let longestSide = max(width, height)+ guard longestSide > maxDimension else { return source }+ let scale = maxDimension / longestSide+ let targetSize = CGSize(+ width: max(1, (width * scale).rounded()),+ height: max(1, (height * scale).rounded())+ )+ let format = UIGraphicsImageRendererFormat.default()+ format.scale = 1+ format.opaque = false+ return UIGraphicsImageRenderer(size: targetSize, format: format)+ .image { _ in+ image.draw(in: CGRect(origin: .zero, size: targetSize))+ }+ .cgImage+ }++ private static let minimumConfidence: Float = 0.3+ private static let regionDetectionLongestSide: CGFloat = 1024+}++private enum RequestKey: Hashable {+ case client(String)+ case anonymous(UUID)+}++private final class RequestState: @unchecked Sendable {+ let key: RequestKey+ var request: VNRequest?+ var isCancelled = false++ init(key: RequestKey) {+ self.key = key+ }+}++private struct OrientedImage {+ let image: UIImage+ let bitmap: CGImage++ var pixelSize: CGSize {+ CGSize(width: bitmap.width, height: bitmap.height)+ }+}++extension VNRectangleObservation {+ fileprivate func polygon(in size: CGSize) -> [CGPoint] {+ [topLeft, topRight, bottomRight, bottomLeft].map { corner in+ CGPoint(x: corner.x * size.width, y: (1 - corner.y) * size.height)+ }+ }+}++private let readingOrderLineBand: CGFloat = 10++extension [RecognizedTextBlock] {+ fileprivate func sortedInReadingOrder() -> [RecognizedTextBlock] {+ sorted { first, second in+ let firstOrigin = first.points.boundsOrigin+ let secondOrigin = second.points.boundsOrigin+ if abs(firstOrigin.y - secondOrigin.y) > readingOrderLineBand {+ return firstOrigin.y < secondOrigin.y+ }+ return firstOrigin.x < secondOrigin.x+ }+ }+}++extension [CGPoint] {+ fileprivate var boundsOrigin: CGPoint {+ CGPoint(x: map(\.x).min() ?? 0, y: map(\.y).min() ?? 0)+ }+}diff --git a/mobile/apps/photos/plugins/ente_vision/ios/ente_vision.podspec b/mobile/apps/photos/plugins/ente_vision/ios/ente_vision.podspecnew file mode 100644index 0000000000..54b577f6aa--- /dev/null+++ b/mobile/apps/photos/plugins/ente_vision/ios/ente_vision.podspec@@ -0,0 +1,14 @@+Pod::Spec.new do |s|+ s.name = 'ente_vision'+ s.version = '0.0.1'+ s.summary = 'Apple Vision text recognition for Ente Photos.'+ s.homepage = 'https://ente.com'+ s.license = { :type => 'AGPL-3.0-only' }+ s.author = { 'Ente' => '[email protected]' }+ s.source = { :path => '.' }+ s.source_files = '{Flutter,Sources}/**/*'+ s.dependency 'Flutter'+ s.platform = :ios, '15.1'+ s.swift_version = '5.0'+ s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES' }+enddiff --git a/mobile/apps/photos/plugins/ente_vision/lib/ente_vision.dart b/mobile/apps/photos/plugins/ente_vision/lib/ente_vision.dartnew file mode 100644index 0000000000..5ca9a64ee1--- /dev/null+++ b/mobile/apps/photos/plugins/ente_vision/lib/ente_vision.dart@@ -0,0 +1 @@+export 'src/vision_text_recognizer.dart';diff --git a/mobile/apps/photos/plugins/ente_vision/lib/src/vision_text_recognizer.dart b/mobile/apps/photos/plugins/ente_vision/lib/src/vision_text_recognizer.dartnew file mode 100644index 0000000000..5e081d7391--- /dev/null+++ b/mobile/apps/photos/plugins/ente_vision/lib/src/vision_text_recognizer.dart@@ -0,0 +1,42 @@+import 'package:flutter/services.dart';++class VisionTextRecognizer {+ VisionTextRecognizer({MethodChannel? methodChannel})+ : _methodChannel = methodChannel ?? const MethodChannel(_methodChannelName);++ static final instance = VisionTextRecognizer();+ static const _methodChannelName = 'io.ente.photos.vision/text_recognition';++ final MethodChannel _methodChannel;++ Future<Map<dynamic, dynamic>> detectText({+ required String imagePath,+ bool includeAllConfidenceScores = false,+ String? requestId,+ }) async {+ final result = await _methodChannel+ .invokeMapMethod<dynamic, dynamic>('textRecognition.detectText', {+ 'imagePath': imagePath,+ 'includeAllConfidenceScores': includeAllConfidenceScores,+ 'requestId': requestId,+ });+ return result ?? const {};+ }++ Future<Map<dynamic, dynamic>> detectTextRegions({+ required String imagePath,+ String? requestId,+ }) async {+ final result = await _methodChannel.invokeMapMethod<dynamic, dynamic>(+ 'textRecognition.detectTextRegions',+ {'imagePath': imagePath, 'requestId': requestId},+ );+ return result ?? const {};+ }++ Future<void> cancelRequest(String requestId) {+ return _methodChannel.invokeMethod<void>('textRecognition.cancelRequest', {+ 'requestId': requestId,+ });+ }+}diff --git a/mobile/apps/photos/plugins/ente_vision/pubspec.yaml b/mobile/apps/photos/plugins/ente_vision/pubspec.yamlnew file mode 100644index 0000000000..c835719ad7--- /dev/null+++ b/mobile/apps/photos/plugins/ente_vision/pubspec.yaml@@ -0,0 +1,22 @@+name: ente_vision+description: Apple Vision text recognition for Ente Photos++resolution: workspace+environment:+ sdk: ">=3.10.0 <4.0.0"+ flutter: ">=3.38.0"++dependencies:+ flutter:+ sdk: flutter++dev_dependencies:+ flutter_lints: 6.0.0+ flutter_test:+ sdk: flutter++flutter:+ plugin:+ platforms:+ ios:+ pluginClass: EnteVisionPlugindiff --git a/mobile/apps/photos/plugins/ente_vision/test/vision_text_recognizer_test.dart b/mobile/apps/photos/plugins/ente_vision/test/vision_text_recognizer_test.dartnew file mode 100644index 0000000000..6f1afcaa13--- /dev/null+++ b/mobile/apps/photos/plugins/ente_vision/test/vision_text_recognizer_test.dart@@ -0,0 +1,133 @@+import 'package:ente_vision/ente_vision.dart';+import 'package:flutter/services.dart';+import 'package:flutter_test/flutter_test.dart';++void main() {+ TestWidgetsFlutterBinding.ensureInitialized();++ const channel = MethodChannel('io.ente.photos.vision/text_recognition');+ final recognizer = VisionTextRecognizer(methodChannel: channel);+ final calls = <MethodCall>[];+ Future<Object?> Function(MethodCall call) reply = (_) async => null;++ setUp(() {+ calls.clear();+ reply = (_) async => null;+ TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger+ .setMockMethodCallHandler(channel, (call) {+ calls.add(call);+ return reply(call);+ });+ });++ tearDown(() {+ TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger+ .setMockMethodCallHandler(channel, null);+ });++ test('detectText sends its arguments and returns the native map', () async {+ final payload = {+ 'blocks': [+ {+ 'text': 'hello',+ 'confidence': 0.9,+ 'points': [+ {'x': 1.0, 'y': 2.0},+ {'x': 10.0, 'y': 2.0},+ {'x': 10.0, 'y': 8.0},+ {'x': 1.0, 'y': 8.0},+ ],+ 'characters': <Object?>[],+ },+ ],+ 'imageWidth': 640,+ 'imageHeight': 480,+ };+ reply = (_) async => payload;++ final result = await recognizer.detectText(+ imagePath: '/tmp/photo.jpg',+ requestId: 'request-1',+ );++ expect(calls.single.method, 'textRecognition.detectText');+ expect(calls.single.arguments, {+ 'imagePath': '/tmp/photo.jpg',+ 'includeAllConfidenceScores': false,+ 'requestId': 'request-1',+ });+ expect(result, payload);+ });++ test('detectText forwards includeAllConfidenceScores', () async {+ await recognizer.detectText(+ imagePath: '/tmp/photo.jpg',+ includeAllConfidenceScores: true,+ );++ expect(calls.single.arguments, {+ 'imagePath': '/tmp/photo.jpg',+ 'includeAllConfidenceScores': true,+ 'requestId': null,+ });+ });++ test(+ 'detectTextRegions sends its arguments and returns the native map',+ () async {+ final payload = {+ 'regions': [+ {+ 'confidence': 0.75,+ 'points': [+ {'x': 0.0, 'y': 0.0},+ {'x': 5.0, 'y': 0.0},+ {'x': 5.0, 'y': 3.0},+ {'x': 0.0, 'y': 3.0},+ ],+ },+ ],+ 'imageWidth': 320,+ 'imageHeight': 240,+ };+ reply = (_) async => payload;++ final result = await recognizer.detectTextRegions(+ imagePath: '/tmp/photo.jpg',+ requestId: 'request-2',+ );++ expect(calls.single.method, 'textRecognition.detectTextRegions');+ expect(calls.single.arguments, {+ 'imagePath': '/tmp/photo.jpg',+ 'requestId': 'request-2',+ });+ expect(result, payload);+ },+ );++ test('cancelRequest sends the request id', () async {+ await recognizer.cancelRequest('request-3');++ expect(calls.single.method, 'textRecognition.cancelRequest');+ expect(calls.single.arguments, {'requestId': 'request-3'});+ });++ test('a null native result becomes an empty map', () async {+ final result = await recognizer.detectText(imagePath: '/tmp/photo.jpg');++ expect(result, isEmpty);+ });++ test('platform exceptions propagate unchanged', () async {+ reply = (_) async =>+ throw PlatformException(code: 'CANCELLED', message: 'cancelled');++ await expectLater(+ recognizer.detectText(imagePath: '/tmp/photo.jpg', requestId: 'r'),+ throwsA(+ isA<PlatformException>().having((e) => e.code, 'code', 'CANCELLED'),+ ),+ );+ });+}diff --git a/mobile/apps/photos/pubspec.yaml b/mobile/apps/photos/pubspec.yamlindex ee093f0b2f..d018cbf951 100644--- a/mobile/apps/photos/pubspec.yaml+++ b/mobile/apps/photos/pubspec.yaml@@ -76,6 +76,8 @@ dependencies:path: ../../packages/stringsente_ui:path: ../../packages/ui+ ente_vision:+ path: plugins/ente_visionequatable: 2.0.7event_bus: 2.0.1exif_reader: 4.2.0diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yamlindex 109b495c9f..efa91aa4a2 100644--- a/mobile/pubspec.yaml+++ b/mobile/pubspec.yaml@@ -12,6 +12,7 @@ workspace:- apps/photos/plugins/ente_crypto- apps/photos/plugins/ente_panorama_viewer- apps/photos/plugins/ente_photos_platform+ - apps/photos/plugins/ente_vision- apps/photos/plugins/media_extension- packages/accounts- packages/account_deletion