Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[rust][mobile][locker] Expose the document scanner over FRB with ORT wiring

laurenspriembbb7909289fe
diff --git a/mobile/apps/locker/android/app/build.gradle b/mobile/apps/locker/android/app/build.gradle
index d8137c171c..e14486bbde 100644
--- a/mobile/apps/locker/android/app/build.gradle
+++ b/mobile/apps/locker/android/app/build.gradle
@@ -12,6 +12,25 @@ if (keystorePropertiesFile.exists()) {
keystoreProperties.load(new FileInputStream(keystorePropertiesFile))
}
+// Ente's custom prebuilt ONNX Runtime AAR is resolved straight from its
+// GitHub release through an Ivy repository; its SHA-256 is pinned in
+// gradle/verification-metadata.xml.
+repositories {
+ exclusiveContent {
+ forRepository {
+ ivy {
+ name = 'enteOnnxRuntimePackaging'
+ url = 'https://github.com/ente/ort-packaging/releases/download'
+ patternLayout {
+ artifact 'ort-[revision]/[artifact]-[revision].[ext]'
+ }
+ metadataSources { artifact() }
+ }
+ }
+ filter { includeGroup 'io.ente.onnxruntime' }
+ }
+}
+
android {
namespace = "io.ente.locker"
compileSdk = 36
@@ -95,4 +114,7 @@ flutter {
dependencies {
// Needed at runtime for Tink since R8 strips these annotations otherwise.
implementation 'com.google.errorprone:error_prone_annotations:2.24.0'
+ // Custom WebGPU/XNNPACK build required by the Rust document scanner,
+ // which dynamically loads libonnxruntime.so.
+ implementation 'io.ente.onnxruntime:onnxruntime-webgpu-android:1.28.0-r3@aar'
}
diff --git a/mobile/apps/locker/android/gradle/verification-metadata.xml b/mobile/apps/locker/android/gradle/verification-metadata.xml
new file mode 100644
index 0000000000..845a72f973
--- /dev/null
+++ b/mobile/apps/locker/android/gradle/verification-metadata.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Gradle dependency verification.
+
+ Every artifact is deliberately trusted except Ente's custom ONNX Runtime
+ binaries: those are fetched from a plain GitHub release (no repository
+ metadata), so their SHA-256 digests are pinned here. The digests come from
+ the SHA256SUMS asset published with the ort-packaging release.
+-->
+<verification-metadata xmlns="https://schema.gradle.org/dependency-verification"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="https://schema.gradle.org/dependency-verification https://schema.gradle.org/dependency-verification/dependency-verification-1.3.xsd">
+ <configuration>
+ <verify-metadata>false</verify-metadata>
+ <verify-signatures>false</verify-signatures>
+ <trusted-artifacts>
+ <trust group="^(?!io\.ente\.onnxruntime$).*$" regex="true"/>
+ </trusted-artifacts>
+ </configuration>
+ <components>
+ <component group="io.ente.onnxruntime" name="onnxruntime-webgpu-android" version="1.28.0-r3">
+ <artifact name="onnxruntime-webgpu-android-1.28.0-r3.aar">
+ <sha256 value="1d268652c6c8392581a06eef889a2823f88ce006633cd76106ca27aade27f193"/>
+ </artifact>
+ </component>
+ </components>
+</verification-metadata>
diff --git a/mobile/apps/locker/ios/Podfile b/mobile/apps/locker/ios/Podfile
index ada3e6f72c..f3608d5344 100644
--- a/mobile/apps/locker/ios/Podfile
+++ b/mobile/apps/locker/ios/Podfile
@@ -1,4 +1,5 @@
-platform :ios, '13.0'
+# ONNX Runtime 1.28 (EnteOnnxRuntime) requires iOS 15.1.
+platform :ios, '15.1'
# CocoaPods analytics sends network stats synchronously affecting flutter build latency.
ENV['COCOAPODS_DISABLE_STATS'] = 'true'
@@ -30,6 +31,11 @@ target 'Runner' do
use_frameworks!
use_modular_headers!
+ # Ente's custom prebuilt ONNX Runtime static libraries (download + SHA-256
+ # verification handled by CocoaPods). Consumed by the ente_locker_frb
+ # build phase, which points the Rust `ort` crate at the platform slice.
+ pod 'EnteOnnxRuntime', :podspec => '../../../../apple/packages/EnteOnnxRuntime/EnteOnnxRuntime.podspec'
+
flutter_install_all_ios_pods File.dirname(File.realpath(__FILE__))
target 'RunnerTests' do
inherit! :search_paths
@@ -43,7 +49,7 @@ post_install do |installer|
installer.pods_project.targets.each do |target|
flutter_additional_ios_build_settings(target)
target.build_configurations.each do |config|
- config.build_settings['IPHONEOS_DEPLOYMENT_TARGET'] = '13.0'
+ config.build_settings['IPHONEOS_DEPLOYMENT_TARGET'] = '15.1'
if target.name == 'ente_locker_frb' && config.name == 'Debug'
config.build_settings['ONLY_ACTIVE_ARCH[sdk=iphonesimulator*]'] = 'YES'
diff --git a/mobile/apps/locker/ios/Podfile.lock b/mobile/apps/locker/ios/Podfile.lock
index 6b7044d706..3b74266e8c 100644
--- a/mobile/apps/locker/ios/Podfile.lock
+++ b/mobile/apps/locker/ios/Podfile.lock
@@ -43,6 +43,7 @@ PODS:
- Flutter
- ente_screen_cover (0.0.1):
- Flutter
+ - EnteOnnxRuntime (1.28.0-r3)
- file_picker (0.0.1):
- DKImagePickerController/PhotoGallery
- Flutter
@@ -98,6 +99,7 @@ DEPENDENCIES:
- ente_locker_frb (from `.symlinks/plugins/ente_locker_frb/ios`)
- ente_mail (from `.symlinks/plugins/ente_mail/ios`)
- ente_screen_cover (from `.symlinks/plugins/ente_screen_cover/ios`)
+ - EnteOnnxRuntime (from `../../../../apple/packages/EnteOnnxRuntime/EnteOnnxRuntime.podspec`)
- file_picker (from `.symlinks/plugins/file_picker/ios`)
- file_saver (from `.symlinks/plugins/file_saver/ios`)
- Flutter (from `Flutter`)
@@ -138,6 +140,8 @@ EXTERNAL SOURCES:
:path: ".symlinks/plugins/ente_mail/ios"
ente_screen_cover:
:path: ".symlinks/plugins/ente_screen_cover/ios"
+ EnteOnnxRuntime:
+ :podspec: "../../../../apple/packages/EnteOnnxRuntime/EnteOnnxRuntime.podspec"
file_picker:
:path: ".symlinks/plugins/file_picker/ios"
file_saver:
@@ -181,9 +185,10 @@ SPEC CHECKSUMS:
device_info_plus: 21fcca2080fbcd348be798aa36c3e5ed849eefbe
DKImagePickerController: 946cec48c7873164274ecc4624d19e3da4c1ef3c
DKPhotoGallery: b3834fecb755ee09a593d7c9e389d8b5d6deed60
- ente_locker_frb: 493a1f7828e127b606d65a484e4fa9171d8c76ec
+ ente_locker_frb: 4509a99affb6399cec6d61bcf7af9676063b091d
ente_mail: 90753dd584f49ee092759a32d4143ca51db69641
ente_screen_cover: 6792566774f1c83939f9c1d03ed0a1f1ded45ae4
+ EnteOnnxRuntime: c994300f7ef809dd318664cfd85868ca69b3cd1a
file_picker: a0560bc09d61de87f12d246fc47d2119e6ef37be
file_saver: 6cdbcddd690cb02b0c1a0c225b37cd805c2bf8b6
Flutter: cabc95a1d2626b1b06e7179b784ebcf0c0cde467
@@ -206,6 +211,6 @@ SPEC CHECKSUMS:
ua_client_hints: 92fe0d139619b73ec9fcb46cc7e079a26178f586
url_launcher_ios: 7a95fa5b60cc718a708b8f2966718e93db0cef1b
-PODFILE CHECKSUM: 1ea06a574e0473671dee0ea3e6eb43a416f8b3d4
+PODFILE CHECKSUM: 05b9a6d67172aa8342d43a3f011879884579e1c5
COCOAPODS: 1.17.0
diff --git a/mobile/apps/locker/ios/Runner.xcodeproj/project.pbxproj b/mobile/apps/locker/ios/Runner.xcodeproj/project.pbxproj
index 72a4909597..c0a56717d0 100644
--- a/mobile/apps/locker/ios/Runner.xcodeproj/project.pbxproj
+++ b/mobile/apps/locker/ios/Runner.xcodeproj/project.pbxproj
@@ -601,7 +601,7 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
- IPHONEOS_DEPLOYMENT_TARGET = 13.0;
+ IPHONEOS_DEPLOYMENT_TARGET = 15.1;
MTL_ENABLE_DEBUG_INFO = NO;
SDKROOT = iphoneos;
SUPPORTED_PLATFORMS = iphoneos;
@@ -622,7 +622,7 @@
DEVELOPMENT_TEAM = 6Z68YJY9Q2;
ENABLE_BITCODE = NO;
INFOPLIST_FILE = Runner/Info.plist;
- IPHONEOS_DEPLOYMENT_TARGET = 13.0;
+ IPHONEOS_DEPLOYMENT_TARGET = 15.1;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
@@ -657,7 +657,7 @@
INFOPLIST_FILE = "Share Extension/Info.plist";
INFOPLIST_KEY_CFBundleDisplayName = "Share Extension";
INFOPLIST_KEY_NSHumanReadableCopyright = "";
- IPHONEOS_DEPLOYMENT_TARGET = 13.0;
+ IPHONEOS_DEPLOYMENT_TARGET = 15.1;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
@@ -700,7 +700,7 @@
INFOPLIST_FILE = "Share Extension/Info.plist";
INFOPLIST_KEY_CFBundleDisplayName = "Share Extension";
INFOPLIST_KEY_NSHumanReadableCopyright = "";
- IPHONEOS_DEPLOYMENT_TARGET = 13.0;
+ IPHONEOS_DEPLOYMENT_TARGET = 15.1;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
@@ -741,7 +741,7 @@
INFOPLIST_FILE = "Share Extension/Info.plist";
INFOPLIST_KEY_CFBundleDisplayName = "Share Extension";
INFOPLIST_KEY_NSHumanReadableCopyright = "";
- IPHONEOS_DEPLOYMENT_TARGET = 13.0;
+ IPHONEOS_DEPLOYMENT_TARGET = 15.1;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
@@ -859,7 +859,7 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
- IPHONEOS_DEPLOYMENT_TARGET = 13.0;
+ IPHONEOS_DEPLOYMENT_TARGET = 15.1;
MTL_ENABLE_DEBUG_INFO = YES;
ONLY_ACTIVE_ARCH = YES;
SDKROOT = iphoneos;
@@ -910,7 +910,7 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
- IPHONEOS_DEPLOYMENT_TARGET = 13.0;
+ IPHONEOS_DEPLOYMENT_TARGET = 15.1;
MTL_ENABLE_DEBUG_INFO = NO;
SDKROOT = iphoneos;
SUPPORTED_PLATFORMS = iphoneos;
@@ -933,7 +933,7 @@
DEVELOPMENT_TEAM = 6Z68YJY9Q2;
ENABLE_BITCODE = NO;
INFOPLIST_FILE = Runner/Info.plist;
- IPHONEOS_DEPLOYMENT_TARGET = 13.0;
+ IPHONEOS_DEPLOYMENT_TARGET = 15.1;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
@@ -960,7 +960,7 @@
DEVELOPMENT_TEAM = 6Z68YJY9Q2;
ENABLE_BITCODE = NO;
INFOPLIST_FILE = Runner/Info.plist;
- IPHONEOS_DEPLOYMENT_TARGET = 13.0;
+ IPHONEOS_DEPLOYMENT_TARGET = 15.1;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
diff --git a/mobile/apps/locker/rust_builder/ios/ente_locker_frb.podspec b/mobile/apps/locker/rust_builder/ios/ente_locker_frb.podspec
index 23d2848722..c288255479 100644
--- a/mobile/apps/locker/rust_builder/ios/ente_locker_frb.podspec
+++ b/mobile/apps/locker/rust_builder/ios/ente_locker_frb.podspec
@@ -20,23 +20,53 @@ A new Flutter FFI plugin project.
s.source = { :path => '.' }
s.source_files = 'Classes/**/*'
s.dependency 'Flutter'
- s.platform = :ios, '13.0'
+ s.platform = :ios, '15.1'
+
+ # Flutter.framework does not contain an i386 slice, and ONNX Runtime 1.28
+ # does not publish an x86_64 iOS Simulator binary.
+ s.user_target_xcconfig = { 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386 x86_64' }
s.swift_version = '5.0'
s.script_phase = {
:name => 'Build Rust library',
- # First argument is relative path to the `rust` folder, second is name of rust library
- :script => 'sh "$(cd -P "$PODS_TARGET_SRCROOT" && pwd)/../../../../cargokit/build_pod.sh" ../../../../../rust/bindings/frb/locker ente_locker_frb',
+ # Point the Rust `ort` crate at the custom prebuilt ONNX Runtime static
+ # archive for the active platform (downloaded and SHA-256 verified by
+ # CocoaPods as the EnteOnnxRuntime pod), then run the Cargokit build.
+ # The build_pod.sh arguments are the relative path to the `rust` folder
+ # and the name of the Rust library.
+ :script => <<-SCRIPT,
+ set -e
+ case "$PLATFORM_NAME" in
+ iphoneos) ort_slice=ios-arm64 ;;
+ iphonesimulator) ort_slice=ios-arm64-simulator ;;
+ *) echo "error: unsupported platform for ONNX Runtime: $PLATFORM_NAME" >&2; exit 1 ;;
+ esac
+ export ORT_LIB_PATH="$PODS_ROOT/EnteOnnxRuntime/onnxruntime.xcframework/$ort_slice"
+ if [ ! -f "$ORT_LIB_PATH/libonnxruntime.a" ]; then
+ echo "error: $ORT_LIB_PATH/libonnxruntime.a is missing; run 'pod install'" >&2
+ exit 1
+ fi
+ unset ORT_IOS_XCFWK_PATH ORT_IOS_XCFWK_LOCATION
+ sh "$(cd -P "$PODS_TARGET_SRCROOT" && pwd)/../../../../cargokit/build_pod.sh" ../../../../../rust/bindings/frb/locker ente_locker_frb
+ SCRIPT
:execution_position => :before_compile,
:input_files => ['${BUILT_PRODUCTS_DIR}/cargokit_phony'],
- # Let XCode know that the static library referenced in -force_load below is
- # created by this build step.
+ # Let Xcode know that the static library linked below is created by this
+ # build step.
:output_files => ["${BUILT_PRODUCTS_DIR}/libente_locker_frb.a"],
}
s.pod_target_xcconfig = {
'DEFINES_MODULE' => 'YES',
- # Flutter.framework does not contain a i386 slice.
- 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386',
- 'OTHER_LDFLAGS' => '-force_load ${BUILT_PRODUCTS_DIR}/libente_locker_frb.a',
+ # ONNX Runtime 1.28 supports only the ARM64 iOS Simulator slice.
+ 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386 x86_64',
+ # Root the primary Flutter Rust Bridge dispatcher so the linker pulls in
+ # the bridge and its transitive FFI exports without force-loading ONNX
+ # Runtime's duplicate protobuf objects.
+ 'OTHER_LDFLAGS' => [
+ '$(inherited)',
+ '${BUILT_PRODUCTS_DIR}/libente_locker_frb.a',
+ '-lc++',
+ '-Wl,-u,_frb_pde_ffi_dispatcher_primary',
+ ].join(' '),
}
end
diff --git a/mobile/apps/locker/rust_builder/macos/ente_locker_frb.podspec b/mobile/apps/locker/rust_builder/macos/ente_locker_frb.podspec
index 749d42c250..47feafffc9 100644
--- a/mobile/apps/locker/rust_builder/macos/ente_locker_frb.podspec
+++ b/mobile/apps/locker/rust_builder/macos/ente_locker_frb.podspec
@@ -22,6 +22,9 @@ A new Flutter FFI plugin project.
s.dependency 'FlutterMacOS'
s.platform = :osx, '10.11'
+ # ONNX Runtime 1.28 does not publish Apple x86_64 binaries, so consumers
+ # must also avoid trying to link an x86_64 application slice.
+ s.user_target_xcconfig = { 'EXCLUDED_ARCHS[sdk=macosx*]' => 'x86_64' }
s.swift_version = '5.0'
s.script_phase = {
@@ -36,6 +39,8 @@ A new Flutter FFI plugin project.
}
s.pod_target_xcconfig = {
'DEFINES_MODULE' => 'YES',
- 'OTHER_LDFLAGS' => '-force_load ${BUILT_PRODUCTS_DIR}/libente_locker_frb.a',
+ # ONNX Runtime 1.28 does not publish Apple x86_64 binaries.
+ 'EXCLUDED_ARCHS[sdk=macosx*]' => 'x86_64',
+ 'OTHER_LDFLAGS' => '-force_load ${BUILT_PRODUCTS_DIR}/libente_locker_frb.a -lc++',
}
end
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 1a9b38fed9..375138ba1d 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2514,9 +2514,11 @@ name = "ente-locker-frb"
version = "0.0.0"
dependencies = [
"ente-accounts",
+ "ente-assets",
"ente-contacts",
"ente-core",
"ente-frb-log",
+ "ente-ml",
"flutter_rust_bridge",
]
diff --git a/rust/bindings/frb/locker/Cargo.toml b/rust/bindings/frb/locker/Cargo.toml
index dffb9ce2a9..a5155678f2 100644
--- a/rust/bindings/frb/locker/Cargo.toml
+++ b/rust/bindings/frb/locker/Cargo.toml
@@ -7,9 +7,11 @@ crate-type = ["cdylib", "staticlib"]
[dependencies]
ente-accounts.workspace = true
+ente-assets.workspace = true
ente-contacts.workspace = true
ente-core.workspace = true
ente-frb-log.workspace = true
+ente-ml.workspace = true
flutter_rust_bridge.workspace = true
[lints.rust]
diff --git a/rust/bindings/frb/locker/src/api/document_scanner_api.rs b/rust/bindings/frb/locker/src/api/document_scanner_api.rs
new file mode 100644
index 0000000000..27f294d788
--- /dev/null
+++ b/rust/bindings/frb/locker/src/api/document_scanner_api.rs
@@ -0,0 +1,348 @@
+use std::any::Any;
+use std::panic::{AssertUnwindSafe, catch_unwind};
+
+use ente_assets::AssetStore;
+use ente_ml::scan;
+use flutter_rust_bridge::frb;
+
+#[derive(Clone, Copy, Debug)]
+pub struct RustPoint {
+ pub x: f64,
+ pub y: f64,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct RustQuad {
+ pub top_left: RustPoint,
+ pub top_right: RustPoint,
+ pub bottom_right: RustPoint,
+ pub bottom_left: RustPoint,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub enum RustColorMode {
+ Color,
+ Grayscale,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct RustPlaneLayout {
+ pub width: i32,
+ pub height: i32,
+ pub y_row_stride: i32,
+ pub uv_row_stride: i32,
+ pub uv_pixel_stride: i32,
+}
+
+#[derive(Clone, Debug)]
+pub struct RustScanOptions {
+ pub color_mode_override: Option<RustColorMode>,
+ pub max_pixels: Option<u32>,
+ /// Must be a multiple of 90.
+ pub rotation_degrees: i32,
+}
+
+#[derive(Clone, Debug)]
+pub struct RustReprocessOptions {
+ /// Same coordinate space as `RustScanResult.quad`: the decoded source
+ /// image.
+ pub quad: RustQuad,
+ /// Must be a multiple of 90.
+ pub rotation_degrees: i32,
+ pub color_mode: RustColorMode,
+ pub max_pixels: Option<u32>,
+ pub jpeg_quality: Option<u8>,
+}
+
+#[derive(Clone, Debug)]
+pub struct RustScanResult {
+ /// In decoded-source coordinates (EXIF applied, before
+ /// `rotation_degrees`); `None` when nothing was detected and the whole
+ /// frame was returned.
+ pub quad: Option<RustQuad>,
+ pub color_mode: RustColorMode,
+ pub output_width: u32,
+ pub output_height: u32,
+ /// The size `quad` is relative to.
+ pub source_width: u32,
+ pub source_height: u32,
+ /// JPEG-encoded.
+ pub processed_image: Vec<u8>,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub enum RustScanErrorKind {
+ InvalidInput,
+ ModelLoad,
+ Codec,
+ Pipeline,
+ /// A panic caught at the FFI boundary.
+ Internal,
+}
+
+#[derive(Clone, Debug)]
+pub struct RustScanError {
+ pub kind: RustScanErrorKind,
+ pub message: String,
+}
+
+/// `Send + Sync`: one session serves both the live preview stream and
+/// capture processing.
+#[frb(opaque)]
+pub struct ScannerSession {
+ inner: scan::ScannerSession,
+}
+
+impl ScannerSession {
+ /// Downloads the model into the asset store at `assets_dir` on first use.
+ pub async fn create(assets_dir: String) -> Result<ScannerSession, RustScanError> {
+ let store = AssetStore::new(assets_dir);
+ let model_path = scan::ensure_segmentation_model(&store)
+ .await
+ .map_err(|error| RustScanError {
+ kind: RustScanErrorKind::ModelLoad,
+ message: error.to_string(),
+ })?;
+ catch_panic(|| {
+ let inner = scan::ScannerSession::new(&model_path.to_string_lossy())?;
+ Ok(ScannerSession { inner })
+ })
+ }
+
+ /// Takes an iOS BGRA8888 frame with `row_stride` bytes per row, so Dart
+ /// passes the camera buffer through unchanged. The returned quad is in
+ /// mask space (256x256), already rotated by `rotation_degrees`: rotate
+ /// the overlay quad, not the frame.
+ pub fn live_detect_bgra(
+ &self,
+ bgra: Vec<u8>,
+ row_stride: i32,
+ width: u32,
+ height: u32,
+ rotation_degrees: i32,
+ ) -> Result<Option<RustQuad>, RustScanError> {
+ catch_panic(AssertUnwindSafe(|| {
+ let rgba = bgra_to_rgba(&bgra, row_stride, width, height)?;
+ let quad = self
+ .inner
+ .live_detect_rgba(&rgba, width, height, rotation_degrees)?;
+ Ok(quad.map(to_api_quad))
+ }))
+ }
+
+ /// Accepts both planar I420 (`uv_pixel_stride == 1`) and interleaved
+ /// NV21/NV12 (`uv_pixel_stride == 2`) YUV_420_888 planes. Quad semantics
+ /// as in [`Self::live_detect_bgra`].
+ pub fn live_detect_yuv420(
+ &self,
+ y: Vec<u8>,
+ u: Vec<u8>,
+ v: Vec<u8>,
+ layout: RustPlaneLayout,
+ rotation_degrees: i32,
+ ) -> Result<Option<RustQuad>, RustScanError> {
+ catch_panic(AssertUnwindSafe(|| {
+ let quad = self.inner.live_detect_yuv420(
+ &y,
+ &u,
+ &v,
+ to_plane_layout(layout),
+ rotation_degrees,
+ )?;
+ Ok(quad.map(to_api_quad))
+ }))
+ }
+
+ pub fn process_capture(
+ &self,
+ image_bytes: Vec<u8>,
+ options: RustScanOptions,
+ ) -> Result<RustScanResult, RustScanError> {
+ catch_panic(AssertUnwindSafe(|| {
+ let result = self
+ .inner
+ .process_capture(&image_bytes, &to_scan_options(&options))?;
+ Ok(to_api_scan_result(result))
+ }))
+ }
+
+ /// No inference runs.
+ pub fn reprocess(
+ &self,
+ source_bytes: Vec<u8>,
+ options: RustReprocessOptions,
+ ) -> Result<RustScanResult, RustScanError> {
+ catch_panic(AssertUnwindSafe(|| {
+ let result = self
+ .inner
+ .reprocess(&source_bytes, &to_reprocess_options(&options))?;
+ Ok(to_api_scan_result(result))
+ }))
+ }
+}
+
+fn catch_panic<T>(body: impl FnOnce() -> Result<T, RustScanError>) -> Result<T, RustScanError> {
+ catch_unwind(AssertUnwindSafe(body)).unwrap_or_else(|panic| {
+ Err(RustScanError {
+ kind: RustScanErrorKind::Internal,
+ message: panic_message(&panic),
+ })
+ })
+}
+
+fn panic_message(panic: &Box<dyn Any + Send>) -> String {
+ if let Some(message) = panic.downcast_ref::<&str>() {
+ (*message).to_string()
+ } else if let Some(message) = panic.downcast_ref::<String>() {
+ message.clone()
+ } else {
+ "unknown panic".to_string()
+ }
+}
+
+fn bgra_to_rgba(
+ bgra: &[u8],
+ row_stride: i32,
+ width: u32,
+ height: u32,
+) -> Result<Vec<u8>, RustScanError> {
+ let invalid = |message: String| RustScanError {
+ kind: RustScanErrorKind::InvalidInput,
+ message,
+ };
+ if width == 0 || height == 0 {
+ return Err(invalid(format!("empty frame: {width}x{height}")));
+ }
+ let width = width as usize;
+ let height = height as usize;
+ let row_stride = usize::try_from(row_stride)
+ .map_err(|_| invalid(format!("negative row stride {row_stride}")))?;
+ let row_bytes = width
+ .checked_mul(4)
+ .ok_or_else(|| invalid(format!("width {width} overflows")))?;
+ if row_stride < row_bytes {
+ return Err(invalid(format!(
+ "row stride {row_stride} is less than {row_bytes} bytes per row"
+ )));
+ }
+ let required = (height - 1)
+ .checked_mul(row_stride)
+ .and_then(|bulk| bulk.checked_add(row_bytes))
+ .ok_or_else(|| invalid(format!("frame {width}x{height} overflows")))?;
+ if bgra.len() < required {
+ return Err(invalid(format!(
+ "buffer holds {} bytes, {required} required",
+ bgra.len()
+ )));
+ }
+
+ let mut rgba = vec![0u8; height * row_bytes];
+ for (row_index, rgba_row) in rgba.chunks_exact_mut(row_bytes).enumerate() {
+ let bgra_row = &bgra[row_index * row_stride..row_index * row_stride + row_bytes];
+ for (rgba_px, bgra_px) in rgba_row.chunks_exact_mut(4).zip(bgra_row.chunks_exact(4)) {
+ rgba_px[0] = bgra_px[2];
+ rgba_px[1] = bgra_px[1];
+ rgba_px[2] = bgra_px[0];
+ rgba_px[3] = bgra_px[3];
+ }
+ }
+ Ok(rgba)
+}
+
+impl From<scan::ScanError> for RustScanError {
+ fn from(value: scan::ScanError) -> Self {
+ let (kind, message) = match value {
+ scan::ScanError::InvalidInput(message) => (RustScanErrorKind::InvalidInput, message),
+ scan::ScanError::ModelLoad(message) => (RustScanErrorKind::ModelLoad, message),
+ scan::ScanError::Codec(message) => (RustScanErrorKind::Codec, message),
+ scan::ScanError::Pipeline(message) => (RustScanErrorKind::Pipeline, message),
+ };
+ RustScanError { kind, message }
+ }
+}
+
+fn to_plane_layout(layout: RustPlaneLayout) -> scan::PlaneLayout {
+ scan::PlaneLayout {
+ width: layout.width,
+ height: layout.height,
+ y_row_stride: layout.y_row_stride,
+ uv_row_stride: layout.uv_row_stride,
+ uv_pixel_stride: layout.uv_pixel_stride,
+ }
+}
+
+fn to_scan_options(options: &RustScanOptions) -> scan::ScanOptions {
+ scan::ScanOptions {
+ color_mode_override: options.color_mode_override.map(to_color_mode),
+ max_pixels: options.max_pixels,
+ rotation_degrees: options.rotation_degrees,
+ }
+}
+
+fn to_reprocess_options(options: &RustReprocessOptions) -> scan::ReprocessOptions {
+ scan::ReprocessOptions {
+ quad: to_quad(options.quad),
+ rotation_degrees: options.rotation_degrees,
+ color_mode: to_color_mode(options.color_mode),
+ max_pixels: options.max_pixels,
+ jpeg_quality: options.jpeg_quality,
+ }
+}
+
+fn to_color_mode(mode: RustColorMode) -> scan::ColorMode {
+ match mode {
+ RustColorMode::Color => scan::ColorMode::Color,
+ RustColorMode::Grayscale => scan::ColorMode::Grayscale,
+ }
+}
+
+fn to_api_color_mode(mode: scan::ColorMode) -> RustColorMode {
+ match mode {
+ scan::ColorMode::Color => RustColorMode::Color,
+ scan::ColorMode::Grayscale => RustColorMode::Grayscale,
+ }
+}
+
+fn to_quad(quad: RustQuad) -> scan::Quad {
+ scan::Quad {
+ top_left: to_point(quad.top_left),
+ top_right: to_point(quad.top_right),
+ bottom_right: to_point(quad.bottom_right),
+ bottom_left: to_point(quad.bottom_left),
+ }
+}
+
+fn to_point(point: RustPoint) -> scan::Point {
+ scan::Point {
+ x: point.x,
+ y: point.y,
+ }
+}
+
+fn to_api_quad(quad: scan::Quad) -> RustQuad {
+ RustQuad {
+ top_left: to_api_point(quad.top_left),
+ top_right: to_api_point(quad.top_right),
+ bottom_right: to_api_point(quad.bottom_right),
+ bottom_left: to_api_point(quad.bottom_left),
+ }
+}
+
+fn to_api_point(point: scan::Point) -> RustPoint {
+ RustPoint {
+ x: point.x,
+ y: point.y,
+ }
+}
+
+fn to_api_scan_result(result: scan::ScanResult) -> RustScanResult {
+ RustScanResult {
+ quad: result.quad.map(to_api_quad),
+ color_mode: to_api_color_mode(result.color_mode),
+ output_width: result.output_width,
+ output_height: result.output_height,
+ source_width: result.source_width,
+ source_height: result.source_height,
+ processed_image: result.processed_image,
+ }
+}
diff --git a/rust/bindings/frb/locker/src/api/mod.rs b/rust/bindings/frb/locker/src/api/mod.rs
index 6bb4a97b24..e8b00b234c 100644
--- a/rust/bindings/frb/locker/src/api/mod.rs
+++ b/rust/bindings/frb/locker/src/api/mod.rs
@@ -1,4 +1,5 @@
pub mod contacts;
+pub mod document_scanner_api;
pub mod init;
#[cfg(any(feature = "flutter", frb_expand))]
pub mod log;