Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[mob][photos] Add ente_vision plugin for Apple Vision OCR

laurenspriem2fbbcbc6517c
diff --git a/mobile/apps/photos/ios/Podfile.lock b/mobile/apps/photos/ios/Podfile.lock
index e5ddacb279..253254105d 100644
--- a/mobile/apps/photos/ios/Podfile.lock
+++ b/mobile/apps/photos/ios/Podfile.lock
@@ -30,6 +30,8 @@ PODS:
- Flutter
- ente_screen_cover (0.0.1):
- Flutter
+ - ente_vision (0.0.1):
+ - Flutter
- EnteOnnxRuntime (1.28.1-r1)
- ffmpeg_kit_custom (6.0.3)
- ffmpeg_kit_flutter (6.0.3):
@@ -255,6 +257,7 @@ DEPENDENCIES:
- ente_qr (from `.symlinks/plugins/ente_qr/ios`)
- ente_screen_brightness (from `.symlinks/plugins/ente_screen_brightness/ios`)
- ente_screen_cover (from `.symlinks/plugins/ente_screen_cover/ios`)
+ - ente_vision (from `.symlinks/plugins/ente_vision/ios`)
- EnteOnnxRuntime (from `../../../../apple/packages/EnteOnnxRuntime/EnteOnnxRuntime.podspec`)
- ffmpeg_kit_flutter (from `.symlinks/plugins/ffmpeg_kit_flutter/ios`)
- file_saver (from `.symlinks/plugins/file_saver/ios`)
@@ -350,6 +353,8 @@ EXTERNAL SOURCES:
:path: ".symlinks/plugins/ente_screen_brightness/ios"
ente_screen_cover:
:path: ".symlinks/plugins/ente_screen_cover/ios"
+ ente_vision:
+ :path: ".symlinks/plugins/ente_vision/ios"
EnteOnnxRuntime:
:podspec: "../../../../apple/packages/EnteOnnxRuntime/EnteOnnxRuntime.podspec"
ffmpeg_kit_flutter:
@@ -451,6 +456,7 @@ SPEC CHECKSUMS:
ente_qr: 86beaf5d2644705db9b565b32dd8b82c05fb36af
ente_screen_brightness: 7dd13c361508ee6098a2578908ebf35e9f567e28
ente_screen_cover: 6792566774f1c83939f9c1d03ed0a1f1ded45ae4
+ ente_vision: 260965fb4a3bda0c0cb72c5f15eb8748fef7c897
EnteOnnxRuntime: 7baa83a95717a6c9c8d0d4599f9f2c90b72d2905
ffmpeg_kit_custom: 682b4f2f1ff1f8abae5a92f6c3540f2441d5be99
ffmpeg_kit_flutter: 915b345acc97d4142e8a9a8549d177ff10f043f5
diff --git a/mobile/apps/photos/ios/Runner.xcodeproj/project.pbxproj b/mobile/apps/photos/ios/Runner.xcodeproj/project.pbxproj
index 1f5453d1d3..ba96b8e348 100644
--- a/mobile/apps/photos/ios/Runner.xcodeproj/project.pbxproj
+++ b/mobile/apps/photos/ios/Runner.xcodeproj/project.pbxproj
@@ -645,6 +645,7 @@
"${BUILT_PRODUCTS_DIR}/ente_qr/ente_qr.framework",
"${BUILT_PRODUCTS_DIR}/ente_screen_brightness/ente_screen_brightness.framework",
"${BUILT_PRODUCTS_DIR}/ente_screen_cover/ente_screen_cover.framework",
+ "${BUILT_PRODUCTS_DIR}/ente_vision/ente_vision.framework",
"${BUILT_PRODUCTS_DIR}/file_saver/file_saver.framework",
"${BUILT_PRODUCTS_DIR}/flutter_image_compress_common/flutter_image_compress_common.framework",
"${BUILT_PRODUCTS_DIR}/flutter_inappwebview_ios/flutter_inappwebview_ios.framework",
@@ -661,7 +662,6 @@
"${BUILT_PRODUCTS_DIR}/launcher_icon_switcher/launcher_icon_switcher.framework",
"${BUILT_PRODUCTS_DIR}/libwebp/libwebp.framework",
"${BUILT_PRODUCTS_DIR}/local_auth_darwin/local_auth_darwin.framework",
- "${BUILT_PRODUCTS_DIR}/media_extension/media_extension.framework",
"${BUILT_PRODUCTS_DIR}/media_kit_libs_ios_video/media_kit_libs_ios_video.framework",
"${BUILT_PRODUCTS_DIR}/media_kit_video/media_kit_video.framework",
"${BUILT_PRODUCTS_DIR}/mobile_ocr/mobile_ocr.framework",
@@ -741,6 +741,7 @@
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/ente_qr.framework",
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/ente_screen_brightness.framework",
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/ente_screen_cover.framework",
+ "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/ente_vision.framework",
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/file_saver.framework",
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/flutter_image_compress_common.framework",
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/flutter_inappwebview_ios.framework",
@@ -757,7 +758,6 @@
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/launcher_icon_switcher.framework",
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/libwebp.framework",
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/local_auth_darwin.framework",
- "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/media_extension.framework",
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/media_kit_libs_ios_video.framework",
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/media_kit_video.framework",
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/mobile_ocr.framework",
diff --git a/mobile/apps/photos/plugins/ente_vision/ios/.swift-format b/mobile/apps/photos/plugins/ente_vision/ios/.swift-format
new file mode 100644
index 0000000000..46b20d5169
--- /dev/null
+++ b/mobile/apps/photos/plugins/ente_vision/ios/.swift-format
@@ -0,0 +1,13 @@
+{
+ "indentation": { "spaces": 4 },
+ "lineLength": 120,
+ "respectsExistingLineBreaks": true,
+ "rules": {
+ "AllPublicDeclarationsHaveDocumentation": false,
+ "AlwaysUseLowerCamelCase": true,
+ "DoNotUseSemicolons": true,
+ "FileScopedDeclarationPrivacy": true,
+ "NeverForceUnwrap": true,
+ "NeverUseImplicitlyUnwrappedOptionals": true
+ }
+}
diff --git a/mobile/apps/photos/plugins/ente_vision/ios/Flutter/EnteVisionPlugin.swift b/mobile/apps/photos/plugins/ente_vision/ios/Flutter/EnteVisionPlugin.swift
new file mode 100644
index 0000000000..9edec741b2
--- /dev/null
+++ b/mobile/apps/photos/plugins/ente_vision/ios/Flutter/EnteVisionPlugin.swift
@@ -0,0 +1,20 @@
+@preconcurrency import Flutter
+import Foundation
+
+@MainActor
+public final class EnteVisionPlugin: NSObject, @preconcurrency FlutterPlugin {
+ private let textRecognitionAdapter: TextRecognitionChannelAdapter
+
+ private init(registrar: FlutterPluginRegistrar) {
+ textRecognitionAdapter = TextRecognitionChannelAdapter(registrar: registrar)
+ super.init()
+ }
+
+ public static func register(with registrar: FlutterPluginRegistrar) {
+ registrar.publish(EnteVisionPlugin(registrar: registrar))
+ }
+
+ public func detachFromEngine(for registrar: FlutterPluginRegistrar) {
+ textRecognitionAdapter.detach()
+ }
+}
diff --git a/mobile/apps/photos/plugins/ente_vision/ios/Flutter/TextRecognitionChannelAdapter.swift b/mobile/apps/photos/plugins/ente_vision/ios/Flutter/TextRecognitionChannelAdapter.swift
new file mode 100644
index 0000000000..76d2a1c6d7
--- /dev/null
+++ b/mobile/apps/photos/plugins/ente_vision/ios/Flutter/TextRecognitionChannelAdapter.swift
@@ -0,0 +1,189 @@
+@preconcurrency import Flutter
+import Foundation
+
+@MainActor
+final class TextRecognitionChannelAdapter {
+ private let channel: FlutterMethodChannel
+ private let recognizer = TextRecognizer()
+ private var isAttached = true
+
+ init(registrar: FlutterPluginRegistrar) {
+ channel = FlutterMethodChannel(
+ name: Self.channelName,
+ binaryMessenger: registrar.messenger()
+ )
+ channel.setMethodCallHandler { [weak self] call, result in
+ self?.handle(call, result: result)
+ }
+ }
+
+ func detach() {
+ guard isAttached else { return }
+ isAttached = false
+ recognizer.cancelAll()
+ channel.setMethodCallHandler(nil)
+ }
+
+ private func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) {
+ guard isAttached else {
+ result(TextRecognitionError.cancelled.flutterError)
+ return
+ }
+ switch call.method {
+ case "textRecognition.detectText":
+ handleDetectText(call, result: result)
+ case "textRecognition.detectTextRegions":
+ handleDetectTextRegions(call, result: result)
+ case "textRecognition.cancelRequest":
+ handleCancelRequest(call, result: result)
+ default:
+ result(FlutterMethodNotImplemented)
+ }
+ }
+
+ private func handleDetectText(_ call: FlutterMethodCall, result: @escaping FlutterResult) {
+ guard let arguments = call.arguments as? [String: Any],
+ let imagePath = arguments["imagePath"] as? String
+ else {
+ result(invalidArgument("imagePath is required"))
+ return
+ }
+ recognizer.recognizeText(
+ imagePath: imagePath,
+ includeAllConfidenceScores: arguments["includeAllConfidenceScores"] as? Bool ?? false,
+ requestId: arguments["requestId"] as? String
+ ) { outcome in
+ deliver(outcome, to: result, channelValue: \.channelValue)
+ }
+ }
+
+ private func handleDetectTextRegions(_ call: FlutterMethodCall, result: @escaping FlutterResult) {
+ guard let arguments = call.arguments as? [String: Any],
+ let imagePath = arguments["imagePath"] as? String
+ else {
+ result(invalidArgument("imagePath is required"))
+ return
+ }
+ recognizer.detectTextRegions(
+ imagePath: imagePath,
+ requestId: arguments["requestId"] as? String
+ ) { outcome in
+ deliver(outcome, to: result, channelValue: \.channelValue)
+ }
+ }
+
+ private func handleCancelRequest(_ call: FlutterMethodCall, result: FlutterResult) {
+ guard let arguments = call.arguments as? [String: Any],
+ let requestId = arguments["requestId"] as? String, !requestId.isEmpty
+ else {
+ result(invalidArgument("requestId is required"))
+ return
+ }
+ recognizer.cancel(requestId: requestId)
+ result(nil)
+ }
+
+ private static let channelName = "io.ente.photos.vision/text_recognition"
+}
+
+private func invalidArgument(_ message: String) -> FlutterError {
+ FlutterError(code: "INVALID_ARGUMENT", message: message, details: nil)
+}
+
+private func deliver<Value: Sendable>(
+ _ outcome: Result<Value, TextRecognitionError>,
+ to result: @escaping FlutterResult,
+ channelValue: @escaping @Sendable (Value) -> [String: Any]
+) {
+ DispatchQueue.main.async {
+ switch outcome {
+ case .success(let value):
+ result(channelValue(value))
+ case .failure(let error):
+ result(error.flutterError)
+ }
+ }
+}
+
+extension TextRecognitionError {
+ fileprivate var flutterError: FlutterError {
+ switch self {
+ case .imageNotFound:
+ FlutterError(code: "IMAGE_NOT_FOUND", message: "Image file does not exist", details: nil)
+ case .imageUndecodable:
+ FlutterError(code: "IMAGE_DECODE_ERROR", message: "Failed to load image from path", details: nil)
+ case .imageBitmapUnavailable:
+ FlutterError(code: "IMAGE_DECODE_ERROR", message: "Failed to get CGImage", details: nil)
+ case .cancelled:
+ FlutterError(code: "CANCELLED", message: "OCR request was cancelled", details: nil)
+ case .detectionFailed(let error):
+ FlutterError(
+ code: "DETECTION_ERROR",
+ message: "Failed to detect text regions",
+ details: error.localizedDescription
+ )
+ case .recognitionFailed(let error):
+ FlutterError(
+ code: "RECOGNITION_ERROR",
+ message: "Text recognition failed",
+ details: error.localizedDescription
+ )
+ }
+ }
+}
+
+extension TextRecognitionResult {
+ fileprivate var channelValue: [String: Any] {
+ [
+ "blocks": blocks.map(\.channelValue),
+ "imageWidth": imageWidth,
+ "imageHeight": imageHeight,
+ ]
+ }
+}
+
+extension RecognizedTextBlock {
+ fileprivate var channelValue: [String: Any] {
+ [
+ "text": text,
+ "confidence": Double(confidence),
+ "points": points.channelValue,
+ "characters": characters.map(\.channelValue),
+ ]
+ }
+}
+
+extension RecognizedCharacter {
+ fileprivate var channelValue: [String: Any] {
+ [
+ "text": text,
+ "confidence": Double(confidence),
+ "points": points.channelValue,
+ ]
+ }
+}
+
+extension TextRegionsResult {
+ fileprivate var channelValue: [String: Any] {
+ [
+ "regions": regions.map(\.channelValue),
+ "imageWidth": imageWidth,
+ "imageHeight": imageHeight,
+ ]
+ }
+}
+
+extension TextRegion {
+ fileprivate var channelValue: [String: Any] {
+ [
+ "confidence": Double(confidence),
+ "points": points.channelValue,
+ ]
+ }
+}
+
+extension [CGPoint] {
+ fileprivate var channelValue: [[String: Double]] {
+ map { ["x": Double($0.x), "y": Double($0.y)] }
+ }
+}
diff --git a/mobile/apps/photos/plugins/ente_vision/ios/Sources/EnteVisionCore/RecognitionLanguageSelector.swift b/mobile/apps/photos/plugins/ente_vision/ios/Sources/EnteVisionCore/RecognitionLanguageSelector.swift
new file mode 100644
index 0000000000..11804a1853
--- /dev/null
+++ b/mobile/apps/photos/plugins/ente_vision/ios/Sources/EnteVisionCore/RecognitionLanguageSelector.swift
@@ -0,0 +1,112 @@
+import Foundation
+
+enum RecognitionLanguageSelector {
+ static func select(
+ preferredLanguages: [String],
+ supportedLanguages: [String],
+ limit: Int = 3
+ ) -> [String] {
+ guard limit > 0, !supportedLanguages.isEmpty else { return [] }
+
+ var selected: [String] = []
+ for preferred in preferredLanguages {
+ let remaining = supportedLanguages.filter { !selected.contains($0) }
+ if let match = bestMatch(for: preferred, in: remaining) {
+ selected.append(match)
+ }
+ if selected.count == limit {
+ return selected
+ }
+ }
+
+ if selected.count < limit,
+ let english = supportedLanguages.first(where: {
+ normalized($0) == "en-us"
+ }),
+ !selected.contains(english)
+ {
+ selected.append(english)
+ }
+
+ return selected.isEmpty
+ ? Array(supportedLanguages.prefix(1))
+ : selected
+ }
+
+ private static func bestMatch(
+ for preferred: String,
+ in supportedLanguages: [String]
+ ) -> String? {
+ if let exact = supportedLanguages.first(where: {
+ normalized($0) == normalized(preferred)
+ }) {
+ return exact
+ }
+
+ let preferredTag = LanguageTag(preferred)
+ guard !preferredTag.language.isEmpty else { return nil }
+ let supportedTags = supportedLanguages.map {
+ (identifier: $0, tag: LanguageTag($0))
+ }
+
+ if let script = preferredTag.effectiveScript,
+ let scriptMatch = supportedTags.first(where: {
+ $0.tag.language == preferredTag.language && $0.tag.effectiveScript == script
+ })
+ {
+ return scriptMatch.identifier
+ }
+
+ if let region = preferredTag.region,
+ let regionMatch = supportedTags.first(where: {
+ $0.tag.language == preferredTag.language && $0.tag.region == region
+ })
+ {
+ return regionMatch.identifier
+ }
+
+ return supportedTags.first(where: {
+ $0.tag.language == preferredTag.language
+ })?.identifier
+ }
+
+ private static func normalized(_ identifier: String) -> String {
+ identifier.replacingOccurrences(of: "_", with: "-").lowercased()
+ }
+}
+
+private struct LanguageTag {
+ let language: String
+ let script: String?
+ let region: String?
+
+ init(_ identifier: String) {
+ let parts =
+ identifier
+ .replacingOccurrences(of: "_", with: "-")
+ .split(separator: "-")
+ .map(String.init)
+ language = parts.first?.lowercased() ?? ""
+ script = parts.dropFirst().first(where: { part in
+ part.count == 4 && part.allSatisfy(\.isLetter)
+ })?.lowercased()
+ region = parts.dropFirst().first(where: { part in
+ (part.count == 2 && part.allSatisfy(\.isLetter)) || (part.count == 3 && part.allSatisfy(\.isNumber))
+ })?.uppercased()
+ }
+
+ var effectiveScript: String? {
+ if let script = script {
+ return script
+ }
+ guard language == "zh", let region = region else { return nil }
+ switch region {
+ case "HK", "MO", "TW":
+ return "hant"
+ case "CN", "MY", "SG":
+ return "hans"
+ default:
+ return nil
+ }
+ }
+}
diff --git a/mobile/apps/photos/plugins/ente_vision/ios/Sources/EnteVisionCore/TextRecognizer.swift b/mobile/apps/photos/plugins/ente_vision/ios/Sources/EnteVisionCore/TextRecognizer.swift
new file mode 100644
index 0000000000..75db29e61c
--- /dev/null
+++ b/mobile/apps/photos/plugins/ente_vision/ios/Sources/EnteVisionCore/TextRecognizer.swift
@@ -0,0 +1,318 @@
+import Foundation
+import UIKit
+import Vision
+
+struct RecognizedCharacter: Sendable {
+ let text: String
+ let confidence: Float
+ let points: [CGPoint]
+}
+
+struct RecognizedTextBlock: Sendable {
+ let text: String
+ let confidence: Float
+ let points: [CGPoint]
+ let characters: [RecognizedCharacter]
+}
+
+struct TextRecognitionResult: Sendable {
+ let blocks: [RecognizedTextBlock]
+ let imageWidth: Int
+ let imageHeight: Int
+}
+
+struct TextRegion: Sendable {
+ let confidence: Float
+ let points: [CGPoint]
+}
+
+struct TextRegionsResult: Sendable {
+ let regions: [TextRegion]
+ let imageWidth: Int
+ let imageHeight: Int
+}
+
+enum TextRecognitionError: Error {
+ case imageNotFound
+ case imageUndecodable
+ case imageBitmapUnavailable
+ case cancelled
+ case detectionFailed(Error)
+ case recognitionFailed(Error)
+}
+
+final class TextRecognizer: @unchecked Sendable {
+ private let lock = NSLock()
+ private var activeRequests: [RequestKey: RequestState] = [:]
+ private let queue = DispatchQueue.global(qos: .userInitiated)
+
+ func recognizeText(
+ imagePath: String,
+ includeAllConfidenceScores: Bool,
+ requestId: String?,
+ completion: @escaping @Sendable (Result<TextRecognitionResult, TextRecognitionError>) -> Void
+ ) {
+ let minimumConfidence: Float = includeAllConfidenceScores ? 0 : Self.minimumConfidence
+ run(requestId: requestId, foreignError: { .recognitionFailed($0) }, completion: completion) {
+ [self] state in
+ let image = try loadOrientedImage(atPath: imagePath, state: state)
+ let request = Self.makeRecognizeTextRequest()
+ try attach(request, to: state)
+ try VNImageRequestHandler(cgImage: image.bitmap, options: [:]).perform([request])
+ let size = image.pixelSize
+ let blocks = (request.results ?? []).compactMap { observation -> RecognizedTextBlock? in
+ guard let candidate = observation.topCandidates(1).first,
+ candidate.confidence >= minimumConfidence
+ else {
+ return nil
+ }
+ return RecognizedTextBlock(
+ text: candidate.string,
+ confidence: candidate.confidence,
+ points: observation.polygon(in: size),
+ characters: Self.characters(of: candidate, in: size)
+ )
+ }
+ return TextRecognitionResult(
+ blocks: blocks.sortedInReadingOrder(),
+ imageWidth: image.bitmap.width,
+ imageHeight: image.bitmap.height
+ )
+ }
+ }
+
+ func detectTextRegions(
+ imagePath: String,
+ requestId: String?,
+ completion: @escaping @Sendable (Result<TextRegionsResult, TextRecognitionError>) -> Void
+ ) {
+ run(requestId: requestId, foreignError: { .detectionFailed($0) }, completion: completion) {
+ [self] state in
+ let image = try loadOrientedImage(atPath: imagePath, state: state)
+ let detectionBitmap =
+ Self.downscaled(image.image, longestSide: Self.regionDetectionLongestSide) ?? image.bitmap
+ let request = VNDetectTextRectanglesRequest()
+ request.reportCharacterBoxes = false
+ try attach(request, to: state)
+ try VNImageRequestHandler(cgImage: detectionBitmap, options: [:]).perform([request])
+ let size = image.pixelSize
+ let regions = (request.results ?? []).map { observation in
+ TextRegion(confidence: observation.confidence, points: observation.polygon(in: size))
+ }
+ return TextRegionsResult(
+ regions: regions,
+ imageWidth: image.bitmap.width,
+ imageHeight: image.bitmap.height
+ )
+ }
+ }
+
+ func cancel(requestId: String) {
+ lock.lock()
+ let state = activeRequests.removeValue(forKey: .client(requestId))
+ state?.isCancelled = true
+ let request = state?.request
+ lock.unlock()
+ request?.cancel()
+ }
+
+ func cancelAll() {
+ lock.lock()
+ let states = Array(activeRequests.values)
+ activeRequests.removeAll()
+ for state in states {
+ state.isCancelled = true
+ }
+ let requests = states.compactMap(\.request)
+ lock.unlock()
+ for request in requests {
+ request.cancel()
+ }
+ }
+
+ private func run<Value>(
+ requestId: String?,
+ foreignError: @escaping @Sendable (Error) -> TextRecognitionError,
+ completion: @escaping @Sendable (Result<Value, TextRecognitionError>) -> Void,
+ _ work: @escaping @Sendable (RequestState) throws -> Value
+ ) {
+ let state = begin(requestId: requestId)
+ queue.async {
+ let outcome = Result { try work(state) }
+ .mapError { $0 as? TextRecognitionError ?? foreignError($0) }
+ completion(self.finish(state, outcome))
+ }
+ }
+
+ private func begin(requestId: String?) -> RequestState {
+ let state = RequestState(key: requestId.map(RequestKey.client) ?? .anonymous(UUID()))
+ lock.lock()
+ let previous = activeRequests.updateValue(state, forKey: state.key)
+ previous?.isCancelled = true
+ let previousRequest = previous?.request
+ lock.unlock()
+ previousRequest?.cancel()
+ return state
+ }
+
+ private func attach(_ request: VNRequest, to state: RequestState) throws {
+ lock.lock()
+ defer { lock.unlock() }
+ guard !state.isCancelled else { throw TextRecognitionError.cancelled }
+ state.request = request
+ }
+
+ private func checkNotCancelled(_ state: RequestState) throws {
+ lock.lock()
+ defer { lock.unlock() }
+ if state.isCancelled {
+ throw TextRecognitionError.cancelled
+ }
+ }
+
+ private func finish<Value>(
+ _ state: RequestState,
+ _ outcome: Result<Value, TextRecognitionError>
+ ) -> Result<Value, TextRecognitionError> {
+ lock.lock()
+ defer { lock.unlock() }
+ if activeRequests[state.key] === state {
+ activeRequests.removeValue(forKey: state.key)
+ }
+ state.request = nil
+ return state.isCancelled ? .failure(.cancelled) : outcome
+ }
+
+ private func loadOrientedImage(atPath path: String, state: RequestState) throws -> OrientedImage {
+ try checkNotCancelled(state)
+ guard FileManager.default.fileExists(atPath: path) else { throw TextRecognitionError.imageNotFound }
+ guard let image = UIImage(contentsOfFile: path) else { throw TextRecognitionError.imageUndecodable }
+ try checkNotCancelled(state)
+ let oriented = Self.orientedUp(image)
+ guard let bitmap = oriented.cgImage else { throw TextRecognitionError.imageBitmapUnavailable }
+ try checkNotCancelled(state)
+ return OrientedImage(image: oriented, bitmap: bitmap)
+ }
+
+ private static func makeRecognizeTextRequest() -> VNRecognizeTextRequest {
+ let request = VNRecognizeTextRequest()
+ request.recognitionLevel = .accurate
+ request.minimumTextHeight = 0.01
+ request.usesLanguageCorrection = true
+ if #available(iOS 16.0, *) {
+ request.automaticallyDetectsLanguage = true
+ request.revision = VNRecognizeTextRequestRevision3
+ } else {
+ request.recognitionLanguages = preferredRecognitionLanguages(for: request)
+ }
+ return request
+ }
+
+ private static func preferredRecognitionLanguages(for request: VNRecognizeTextRequest) -> [String] {
+ let supported = (try? request.supportedRecognitionLanguages()) ?? ["en-US"]
+ return RecognitionLanguageSelector.select(
+ preferredLanguages: Locale.preferredLanguages,
+ supportedLanguages: supported
+ )
+ }
+
+ private static func characters(of candidate: VNRecognizedText, in size: CGSize) -> [RecognizedCharacter] {
+ let text = candidate.string
+ return text.indices.compactMap { start in
+ let range = start..<text.index(after: start)
+ guard let box = try? candidate.boundingBox(for: range) else { return nil }
+ return RecognizedCharacter(
+ text: String(text[range]),
+ confidence: candidate.confidence,
+ points: box.polygon(in: size)
+ )
+ }
+ }
+
+ private static func orientedUp(_ image: UIImage) -> UIImage {
+ guard image.imageOrientation != .up else { return image }
+ let format = UIGraphicsImageRendererFormat.default()
+ format.scale = image.scale
+ format.opaque = false
+ return UIGraphicsImageRenderer(size: image.size, format: format).image { _ in
+ image.draw(in: CGRect(origin: .zero, size: image.size))
+ }
+ }
+
+ private static func downscaled(_ image: UIImage, longestSide maxDimension: CGFloat) -> CGImage? {
+ guard let source = image.cgImage else { return nil }
+ let width = CGFloat(source.width)
+ let height = CGFloat(source.height)
+ let longestSide = max(width, height)
+ guard longestSide > maxDimension else { return source }
+ let scale = maxDimension / longestSide
+ let targetSize = CGSize(
+ width: max(1, (width * scale).rounded()),
+ height: max(1, (height * scale).rounded())
+ )
+ let format = UIGraphicsImageRendererFormat.default()
+ format.scale = 1
+ format.opaque = false
+ return UIGraphicsImageRenderer(size: targetSize, format: format)
+ .image { _ in
+ image.draw(in: CGRect(origin: .zero, size: targetSize))
+ }
+ .cgImage
+ }
+
+ private static let minimumConfidence: Float = 0.3
+ private static let regionDetectionLongestSide: CGFloat = 1024
+}
+
+private enum RequestKey: Hashable {
+ case client(String)
+ case anonymous(UUID)
+}
+
+private final class RequestState: @unchecked Sendable {
+ let key: RequestKey
+ var request: VNRequest?
+ var isCancelled = false
+
+ init(key: RequestKey) {
+ self.key = key
+ }
+}
+
+private struct OrientedImage {
+ let image: UIImage
+ let bitmap: CGImage
+
+ var pixelSize: CGSize {
+ CGSize(width: bitmap.width, height: bitmap.height)
+ }
+}
+
+extension VNRectangleObservation {
+ fileprivate func polygon(in size: CGSize) -> [CGPoint] {
+ [topLeft, topRight, bottomRight, bottomLeft].map { corner in
+ CGPoint(x: corner.x * size.width, y: (1 - corner.y) * size.height)
+ }
+ }
+}
+
+private let readingOrderLineBand: CGFloat = 10
+
+extension [RecognizedTextBlock] {
+ fileprivate func sortedInReadingOrder() -> [RecognizedTextBlock] {
+ sorted { first, second in
+ let firstOrigin = first.points.boundsOrigin
+ let secondOrigin = second.points.boundsOrigin
+ if abs(firstOrigin.y - secondOrigin.y) > readingOrderLineBand {
+ return firstOrigin.y < secondOrigin.y
+ }
+ return firstOrigin.x < secondOrigin.x
+ }
+ }
+}
+
+extension [CGPoint] {
+ fileprivate var boundsOrigin: CGPoint {
+ CGPoint(x: map(\.x).min() ?? 0, y: map(\.y).min() ?? 0)
+ }
+}
diff --git a/mobile/apps/photos/plugins/ente_vision/ios/ente_vision.podspec b/mobile/apps/photos/plugins/ente_vision/ios/ente_vision.podspec
new file mode 100644
index 0000000000..54b577f6aa
--- /dev/null
+++ b/mobile/apps/photos/plugins/ente_vision/ios/ente_vision.podspec
@@ -0,0 +1,14 @@
+Pod::Spec.new do |s|
+ s.name = 'ente_vision'
+ s.version = '0.0.1'
+ s.summary = 'Apple Vision text recognition for Ente Photos.'
+ s.homepage = 'https://ente.com'
+ s.license = { :type => 'AGPL-3.0-only' }
+ s.author = { 'Ente' => '[email protected]' }
+ s.source = { :path => '.' }
+ s.source_files = '{Flutter,Sources}/**/*'
+ s.dependency 'Flutter'
+ s.platform = :ios, '15.1'
+ s.swift_version = '5.0'
+ s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES' }
+end
diff --git a/mobile/apps/photos/plugins/ente_vision/lib/ente_vision.dart b/mobile/apps/photos/plugins/ente_vision/lib/ente_vision.dart
new file mode 100644
index 0000000000..5ca9a64ee1
--- /dev/null
+++ b/mobile/apps/photos/plugins/ente_vision/lib/ente_vision.dart
@@ -0,0 +1 @@
+export 'src/vision_text_recognizer.dart';
diff --git a/mobile/apps/photos/plugins/ente_vision/lib/src/vision_text_recognizer.dart b/mobile/apps/photos/plugins/ente_vision/lib/src/vision_text_recognizer.dart
new file mode 100644
index 0000000000..5e081d7391
--- /dev/null
+++ b/mobile/apps/photos/plugins/ente_vision/lib/src/vision_text_recognizer.dart
@@ -0,0 +1,42 @@
+import 'package:flutter/services.dart';
+
+class VisionTextRecognizer {
+ VisionTextRecognizer({MethodChannel? methodChannel})
+ : _methodChannel = methodChannel ?? const MethodChannel(_methodChannelName);
+
+ static final instance = VisionTextRecognizer();
+ static const _methodChannelName = 'io.ente.photos.vision/text_recognition';
+
+ final MethodChannel _methodChannel;
+
+ Future<Map<dynamic, dynamic>> detectText({
+ required String imagePath,
+ bool includeAllConfidenceScores = false,
+ String? requestId,
+ }) async {
+ final result = await _methodChannel
+ .invokeMapMethod<dynamic, dynamic>('textRecognition.detectText', {
+ 'imagePath': imagePath,
+ 'includeAllConfidenceScores': includeAllConfidenceScores,
+ 'requestId': requestId,
+ });
+ return result ?? const {};
+ }
+
+ Future<Map<dynamic, dynamic>> detectTextRegions({
+ required String imagePath,
+ String? requestId,
+ }) async {
+ final result = await _methodChannel.invokeMapMethod<dynamic, dynamic>(
+ 'textRecognition.detectTextRegions',
+ {'imagePath': imagePath, 'requestId': requestId},
+ );
+ return result ?? const {};
+ }
+
+ Future<void> cancelRequest(String requestId) {
+ return _methodChannel.invokeMethod<void>('textRecognition.cancelRequest', {
+ 'requestId': requestId,
+ });
+ }
+}
diff --git a/mobile/apps/photos/plugins/ente_vision/pubspec.yaml b/mobile/apps/photos/plugins/ente_vision/pubspec.yaml
new file mode 100644
index 0000000000..c835719ad7
--- /dev/null
+++ b/mobile/apps/photos/plugins/ente_vision/pubspec.yaml
@@ -0,0 +1,22 @@
+name: ente_vision
+description: Apple Vision text recognition for Ente Photos
+
+resolution: workspace
+environment:
+ sdk: ">=3.10.0 <4.0.0"
+ flutter: ">=3.38.0"
+
+dependencies:
+ flutter:
+ sdk: flutter
+
+dev_dependencies:
+ flutter_lints: 6.0.0
+ flutter_test:
+ sdk: flutter
+
+flutter:
+ plugin:
+ platforms:
+ ios:
+ pluginClass: EnteVisionPlugin
diff --git a/mobile/apps/photos/plugins/ente_vision/test/vision_text_recognizer_test.dart b/mobile/apps/photos/plugins/ente_vision/test/vision_text_recognizer_test.dart
new file mode 100644
index 0000000000..6f1afcaa13
--- /dev/null
+++ b/mobile/apps/photos/plugins/ente_vision/test/vision_text_recognizer_test.dart
@@ -0,0 +1,133 @@
+import 'package:ente_vision/ente_vision.dart';
+import 'package:flutter/services.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+void main() {
+ TestWidgetsFlutterBinding.ensureInitialized();
+
+ const channel = MethodChannel('io.ente.photos.vision/text_recognition');
+ final recognizer = VisionTextRecognizer(methodChannel: channel);
+ final calls = <MethodCall>[];
+ Future<Object?> Function(MethodCall call) reply = (_) async => null;
+
+ setUp(() {
+ calls.clear();
+ reply = (_) async => null;
+ TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger
+ .setMockMethodCallHandler(channel, (call) {
+ calls.add(call);
+ return reply(call);
+ });
+ });
+
+ tearDown(() {
+ TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger
+ .setMockMethodCallHandler(channel, null);
+ });
+
+ test('detectText sends its arguments and returns the native map', () async {
+ final payload = {
+ 'blocks': [
+ {
+ 'text': 'hello',
+ 'confidence': 0.9,
+ 'points': [
+ {'x': 1.0, 'y': 2.0},
+ {'x': 10.0, 'y': 2.0},
+ {'x': 10.0, 'y': 8.0},
+ {'x': 1.0, 'y': 8.0},
+ ],
+ 'characters': <Object?>[],
+ },
+ ],
+ 'imageWidth': 640,
+ 'imageHeight': 480,
+ };
+ reply = (_) async => payload;
+
+ final result = await recognizer.detectText(
+ imagePath: '/tmp/photo.jpg',
+ requestId: 'request-1',
+ );
+
+ expect(calls.single.method, 'textRecognition.detectText');
+ expect(calls.single.arguments, {
+ 'imagePath': '/tmp/photo.jpg',
+ 'includeAllConfidenceScores': false,
+ 'requestId': 'request-1',
+ });
+ expect(result, payload);
+ });
+
+ test('detectText forwards includeAllConfidenceScores', () async {
+ await recognizer.detectText(
+ imagePath: '/tmp/photo.jpg',
+ includeAllConfidenceScores: true,
+ );
+
+ expect(calls.single.arguments, {
+ 'imagePath': '/tmp/photo.jpg',
+ 'includeAllConfidenceScores': true,
+ 'requestId': null,
+ });
+ });
+
+ test(
+ 'detectTextRegions sends its arguments and returns the native map',
+ () async {
+ final payload = {
+ 'regions': [
+ {
+ 'confidence': 0.75,
+ 'points': [
+ {'x': 0.0, 'y': 0.0},
+ {'x': 5.0, 'y': 0.0},
+ {'x': 5.0, 'y': 3.0},
+ {'x': 0.0, 'y': 3.0},
+ ],
+ },
+ ],
+ 'imageWidth': 320,
+ 'imageHeight': 240,
+ };
+ reply = (_) async => payload;
+
+ final result = await recognizer.detectTextRegions(
+ imagePath: '/tmp/photo.jpg',
+ requestId: 'request-2',
+ );
+
+ expect(calls.single.method, 'textRecognition.detectTextRegions');
+ expect(calls.single.arguments, {
+ 'imagePath': '/tmp/photo.jpg',
+ 'requestId': 'request-2',
+ });
+ expect(result, payload);
+ },
+ );
+
+ test('cancelRequest sends the request id', () async {
+ await recognizer.cancelRequest('request-3');
+
+ expect(calls.single.method, 'textRecognition.cancelRequest');
+ expect(calls.single.arguments, {'requestId': 'request-3'});
+ });
+
+ test('a null native result becomes an empty map', () async {
+ final result = await recognizer.detectText(imagePath: '/tmp/photo.jpg');
+
+ expect(result, isEmpty);
+ });
+
+ test('platform exceptions propagate unchanged', () async {
+ reply = (_) async =>
+ throw PlatformException(code: 'CANCELLED', message: 'cancelled');
+
+ await expectLater(
+ recognizer.detectText(imagePath: '/tmp/photo.jpg', requestId: 'r'),
+ throwsA(
+ isA<PlatformException>().having((e) => e.code, 'code', 'CANCELLED'),
+ ),
+ );
+ });
+}
diff --git a/mobile/apps/photos/pubspec.yaml b/mobile/apps/photos/pubspec.yaml
index ee093f0b2f..d018cbf951 100644
--- a/mobile/apps/photos/pubspec.yaml
+++ b/mobile/apps/photos/pubspec.yaml
@@ -76,6 +76,8 @@ dependencies:
path: ../../packages/strings
ente_ui:
path: ../../packages/ui
+ ente_vision:
+ path: plugins/ente_vision
equatable: 2.0.7
event_bus: 2.0.1
exif_reader: 4.2.0
diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml
index 109b495c9f..efa91aa4a2 100644
--- a/mobile/pubspec.yaml
+++ b/mobile/pubspec.yaml
@@ -12,6 +12,7 @@ workspace:
- apps/photos/plugins/ente_crypto
- apps/photos/plugins/ente_panorama_viewer
- apps/photos/plugins/ente_photos_platform
+ - apps/photos/plugins/ente_vision
- apps/photos/plugins/media_extension
- packages/accounts
- packages/account_deletion