Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[locker] Preserve detected scanner crops through review and export

laurenspriem0ad98647efaa
diff --git a/mobile/apps/locker/changes/preserve-scanner-crop.md b/mobile/apps/locker/changes/preserve-scanner-crop.md
new file mode 100644
index 0000000000..7776fd850c
--- /dev/null
+++ b/mobile/apps/locker/changes/preserve-scanner-crop.md
@@ -0,0 +1 @@
+- Keep supported camera crops when reviewing and saving scans, including retakes. Ask for crop review when the photo cannot support the detected edges.
diff --git a/mobile/apps/locker/lib/services/scanner/document_scanner_service.dart b/mobile/apps/locker/lib/services/scanner/document_scanner_service.dart
index ca543a5165..811ca3a47c 100644
--- a/mobile/apps/locker/lib/services/scanner/document_scanner_service.dart
+++ b/mobile/apps/locker/lib/services/scanner/document_scanner_service.dart
@@ -137,11 +137,21 @@ class DocumentScannerService {
_logger.warning('Live detection frame failed', error);
}
- Future<ScannedPage> processCapture(Uint8List capturedJpeg) async {
+ Future<ScannedPage> processCapture(
+ Uint8List capturedJpeg, {
+ ScanCaptureRegion? region,
+ }) async {
try {
final result = await _readySession.processCapture(
imageBytes: capturedJpeg,
maxPixels: _maxPixels,
+ region: region == null
+ ? null
+ : RustCaptureRegion(
+ normalizedQuad: _inSourcePixels(region.quad, 1, 1),
+ frameWidth: region.frameSize.width.round(),
+ frameHeight: region.frameSize.height.round(),
+ ),
);
final id = '${DateTime.now().microsecondsSinceEpoch}_${_counter++}';
final source = File(p.join(_root.path, '${id}_src.jpg'));
@@ -166,6 +176,7 @@ class DocumentScannerService {
sourceHeight: result.sourceHeight,
width: result.outputWidth,
height: result.outputHeight,
+ needsCropReview: result.needsCropReview,
);
} catch (e, s) {
_logger.severe('processCapture failed', e, s);
@@ -206,6 +217,7 @@ class DocumentScannerService {
rotationDegrees: rotationDegrees,
width: result.outputWidth,
height: result.outputHeight,
+ needsCropReview: quad == null ? page.needsCropReview : false,
);
} catch (e, s) {
_logger.severe('updatePage failed', e, s);
diff --git a/mobile/apps/locker/lib/services/scanner/scan_session_controller.dart b/mobile/apps/locker/lib/services/scanner/scan_session_controller.dart
index 70ca7b49a8..94a3f1e1bf 100644
--- a/mobile/apps/locker/lib/services/scanner/scan_session_controller.dart
+++ b/mobile/apps/locker/lib/services/scanner/scan_session_controller.dart
@@ -88,12 +88,15 @@ class ScanSessionController extends ChangeNotifier {
) =>
_service.detectLiveBgra(bytes, rowStride, width, height, rotationDegrees);
- void addCapture(Uint8List capturedJpeg) {
+ void addCapture(Uint8List capturedJpeg, {ScanCaptureRegion? region}) {
_pendingCount++;
notifyListeners();
_queue = _queue.then((_) async {
try {
- final page = await _service.processCapture(capturedJpeg);
+ final page = await _service.processCapture(
+ capturedJpeg,
+ region: region,
+ );
if (_disposed) {
await _service.disposePage(page);
return;
@@ -109,12 +112,19 @@ class ScanSessionController extends ChangeNotifier {
});
}
- Future<ScannedPage?> replaceCapture(String pageId, Uint8List capturedJpeg) {
+ Future<ScannedPage?> replaceCapture(
+ String pageId,
+ Uint8List capturedJpeg, {
+ ScanCaptureRegion? region,
+ }) {
_pendingCount++;
notifyListeners();
final task = _queue.then<ScannedPage?>((_) async {
try {
- final replacement = await _service.processCapture(capturedJpeg);
+ final replacement = await _service.processCapture(
+ capturedJpeg,
+ region: region,
+ );
final index = _pages.indexWhere((page) => page.id == pageId);
if (_disposed || index < 0) {
await _service.disposePage(replacement);
@@ -195,12 +205,17 @@ class ScanSessionController extends ChangeNotifier {
}
Future<File> buildPdf() async {
+ await waitForPending();
+ final pages = List.of(_pages);
+ if (pages.any((page) => page.needsCropReview)) {
+ throw StateError('Document crops must be reviewed before export');
+ }
final fileName = _fileName;
if (fileName == null) {
throw StateError('fileName has not been set');
}
final specs = <PdfPageSpec>[];
- for (final page in List.of(_pages)) {
+ for (final page in pages) {
final jpeg = await page.processedJpeg.readAsBytes();
specs.add(
PdfPageSpec(
diff --git a/mobile/apps/locker/lib/services/scanner/scanner_models.dart b/mobile/apps/locker/lib/services/scanner/scanner_models.dart
index b23bbd7ada..0cebf26d74 100644
--- a/mobile/apps/locker/lib/services/scanner/scanner_models.dart
+++ b/mobile/apps/locker/lib/services/scanner/scanner_models.dart
@@ -4,6 +4,13 @@ import 'dart:ui';
enum ScanColorMode { color, grayscale }
+class ScanCaptureRegion {
+ const ScanCaptureRegion({required this.quad, required this.frameSize});
+
+ final ScanQuad quad;
+ final Size frameSize;
+}
+
class ScanQuad {
ScanQuad._(List<Offset> corners) : corners = List.unmodifiable(corners);
@@ -179,6 +186,7 @@ class ScannedPage {
required this.sourceHeight,
required this.width,
required this.height,
+ this.needsCropReview = false,
});
final String id;
@@ -189,6 +197,8 @@ class ScannedPage {
final ScanQuad quad;
+ final bool needsCropReview;
+
final int rotationDegrees;
final ScanColorMode resolvedColorMode;
@@ -206,11 +216,13 @@ class ScannedPage {
ScanColorMode? resolvedColorMode,
int? width,
int? height,
+ bool? needsCropReview,
}) => ScannedPage(
id: id,
sourceJpeg: sourceJpeg,
processedJpeg: processedJpeg ?? this.processedJpeg,
quad: quad ?? this.quad,
+ needsCropReview: needsCropReview ?? this.needsCropReview,
rotationDegrees: rotationDegrees ?? this.rotationDegrees,
resolvedColorMode: resolvedColorMode ?? this.resolvedColorMode,
sourceWidth: sourceWidth,
diff --git a/mobile/apps/locker/lib/ui/pages/scanner/scanner_capture_page.dart b/mobile/apps/locker/lib/ui/pages/scanner/scanner_capture_page.dart
index ff3e91cd0f..c3071074fb 100644
--- a/mobile/apps/locker/lib/ui/pages/scanner/scanner_capture_page.dart
+++ b/mobile/apps/locker/lib/ui/pages/scanner/scanner_capture_page.dart
@@ -43,7 +43,9 @@ class _ScannerCapturePageState extends State<ScannerCapturePage>
CameraController? _camera;
_CameraStatus _status = _CameraStatus.starting;
- ScanQuad? _stableQuad;
+ ScanCaptureRegion? _stableRegion;
+ Size? _liveFrameSize;
+ ScanQuad? get _stableQuad => _stableRegion?.quad;
bool _analysisInFlight = false;
Duration? _analysisObservedAt;
bool _takingPicture = false;
@@ -139,7 +141,7 @@ class _ScannerCapturePageState extends State<ScannerCapturePage>
_resetLiveTracking();
setState(() {
_status = _CameraStatus.starting;
- _stableQuad = null;
+ _stableRegion = null;
});
_autoCapture.reset();
try {
@@ -201,7 +203,7 @@ class _ScannerCapturePageState extends State<ScannerCapturePage>
if (camera == null) return;
setState(() {
_camera = null;
- _stableQuad = null;
+ _stableRegion = null;
});
_autoCapture.reset();
await camera.dispose();
@@ -261,6 +263,14 @@ class _ScannerCapturePageState extends State<ScannerCapturePage>
return;
}
final now = _frameClock.elapsed;
+ final frameSize = rotation % 180 == 0
+ ? Size(image.width.toDouble(), image.height.toDouble())
+ : Size(image.height.toDouble(), image.width.toDouble());
+ if (_liveFrameSize != null && _liveFrameSize != frameSize) {
+ _stabilizer.reset();
+ _autoCapture.invalidateArming();
+ }
+ _liveFrameSize = frameSize;
final sample = _stabilizer.update(
raw,
observedAt: observedAt,
@@ -282,7 +292,12 @@ class _ScannerCapturePageState extends State<ScannerCapturePage>
documentPresent: raw != null,
);
}
- setState(() => _stableQuad = sample.displayQuad);
+ setState(() {
+ final quad = sample.displayQuad;
+ _stableRegion = quad == null
+ ? null
+ : ScanCaptureRegion(quad: quad, frameSize: frameSize);
+ });
if (fire) unawaited(_capture());
} catch (_) {
if (mounted && generation == _analysisGeneration && camera == _camera) {
@@ -307,14 +322,17 @@ class _ScannerCapturePageState extends State<ScannerCapturePage>
return;
}
}
- setState(() => _stableQuad = null);
+ setState(() {
+ _stableRegion = null;
+ });
}
void _resetLiveTracking() {
_analysisGeneration++;
_quadExpiry?.cancel();
_stabilizer.reset(observedBefore: _frameClock.elapsed);
- _stableQuad = null;
+ _stableRegion = null;
+ _liveFrameSize = null;
}
Future<void> _capture() async {
@@ -323,12 +341,13 @@ class _ScannerCapturePageState extends State<ScannerCapturePage>
return;
}
final retakePageId = _retakePageId;
- final quad = _stableQuad ?? ScanQuad.fullFrame();
+ final region = _stableRegion;
+ final quad = region?.quad ?? ScanQuad.fullFrame();
_autoCapture.notifyCaptureStarted();
unawaited(HapticFeedback.mediumImpact());
setState(() {
_takingPicture = true;
- _stableQuad = null;
+ _stableRegion = null;
_snapQuad = quad;
_snapId++;
});
@@ -338,9 +357,13 @@ class _ScannerCapturePageState extends State<ScannerCapturePage>
final bytes = await shot.readAsBytes();
unawaited(_deleteQuietly(File(shot.path)));
if (retakePageId == null) {
- _onShotTaken(bytes, quad);
+ _onShotTaken(bytes, region);
} else {
- final replacement = await _session.replaceCapture(retakePageId, bytes);
+ final replacement = await _session.replaceCapture(
+ retakePageId,
+ bytes,
+ region: region,
+ );
if (!mounted || replacement == null) return;
unawaited(_openReview());
}
@@ -360,11 +383,13 @@ class _ScannerCapturePageState extends State<ScannerCapturePage>
} catch (_) {}
}
- void _onShotTaken(Uint8List bytes, ScanQuad quad) {
+ void _onShotTaken(Uint8List bytes, ScanCaptureRegion? region) {
final capture = _PendingCapture(_captureSeq++);
_pending.add(capture);
- _session.addCapture(bytes);
- unawaited(_launchFlight(capture, bytes, quad));
+ _session.addCapture(bytes, region: region);
+ unawaited(
+ _launchFlight(capture, bytes, region?.quad ?? ScanQuad.fullFrame()),
+ );
}
Future<void> _launchFlight(
@@ -469,12 +494,8 @@ class _ScannerCapturePageState extends State<ScannerCapturePage>
}
void _purgeResolved() {
- _PendingCapture? newestLanded;
- for (final capture in _pending) {
- if (capture.landed) newestLanded = capture;
- }
for (final capture in _pending) {
- if (capture.landed && (capture.failed || capture != newestLanded)) {
+ if (capture.landed && capture.resolved) {
_releaseSnapshot(capture);
}
}
diff --git a/mobile/apps/locker/lib/ui/pages/scanner/scanner_review_page.dart b/mobile/apps/locker/lib/ui/pages/scanner/scanner_review_page.dart
index 5b8e9e0d46..825da6caaf 100644
--- a/mobile/apps/locker/lib/ui/pages/scanner/scanner_review_page.dart
+++ b/mobile/apps/locker/lib/ui/pages/scanner/scanner_review_page.dart
@@ -166,8 +166,32 @@ class _ScannerReviewPageState extends State<ScannerReviewPage>
return pages[_index.clamp(0, pages.length - 1)];
}
- Future<File> _buildPdf() async {
+ Future<File?> _buildPdf() async {
await widget.session.waitForPending();
+ if (!mounted) return null;
+ for (final page in widget.session.pages.where(
+ (page) => page.needsCropReview,
+ )) {
+ if (_pageController.hasClients) {
+ _jumpToPage(
+ widget.session.pages.indexWhere(
+ (candidate) => candidate.id == page.id,
+ ),
+ );
+ }
+ await Navigator.of(context).push(
+ MaterialPageRoute(
+ builder: (_) =>
+ ScannerCropPage(session: widget.session, pageId: page.id),
+ ),
+ );
+ if (!mounted ||
+ widget.session.pages.any(
+ (candidate) => candidate.id == page.id && candidate.needsCropReview,
+ )) {
+ return null;
+ }
+ }
return widget.session.buildPdf();
}
@@ -183,7 +207,7 @@ class _ScannerReviewPageState extends State<ScannerReviewPage>
Future<void> _saveToEnte() => _runExclusive(() async {
final pdf = await _buildPdf();
- if (!mounted) return;
+ if (!mounted || pdf == null) return;
final didUpload = await widget.onUploadFiles([pdf]);
if (!mounted) return;
if (didUpload) {
@@ -194,6 +218,7 @@ class _ScannerReviewPageState extends State<ScannerReviewPage>
Future<void> _share() => _runExclusive(() async {
final pdf = await _buildPdf();
+ if (!mounted || pdf == null) return;
await SharePlus.instance.share(ShareParams(files: [XFile(pdf.path)]));
});
@@ -578,6 +603,18 @@ class _ScannerReviewPageState extends State<ScannerReviewPage>
},
),
),
+ if (pageCount > 0 && pages[current].needsCropReview)
+ Padding(
+ padding: const EdgeInsets.symmetric(
+ horizontal: Spacing.lg,
+ vertical: Spacing.sm,
+ ),
+ child: Text(
+ l10n.scannerCropReviewRequired,
+ textAlign: TextAlign.center,
+ style: TextStyles.mini.copyWith(color: colors.warning),
+ ),
+ ),
if (pageCount > 0)
Text(
l10n.scanPageOfTotal(
diff --git a/mobile/packages/strings/lib/l10n/arb/strings_en.arb b/mobile/packages/strings/lib/l10n/arb/strings_en.arb
index 9cbfb6e6b2..6f4a38308a 100644
--- a/mobile/packages/strings/lib/l10n/arb/strings_en.arb
+++ b/mobile/packages/strings/lib/l10n/arb/strings_en.arb
@@ -3725,6 +3725,7 @@
"scannerTorchOn": "Turn on torch",
"scannerCapture": "Capture",
"scannerPreparing": "Preparing scanner...",
+ "scannerCropReviewRequired": "Check the crop before saving to avoid cutting off part of the document.",
"scanProcessing": "Processing your scan...",
"scanPageOfTotal": "Page {current} of {total}",
"@scanPageOfTotal": {
diff --git a/rust/bindings/frb/locker/src/api/document_scanner_api.rs b/rust/bindings/frb/locker/src/api/document_scanner_api.rs
index 8d1e3d393f..4178e09380 100644
--- a/rust/bindings/frb/locker/src/api/document_scanner_api.rs
+++ b/rust/bindings/frb/locker/src/api/document_scanner_api.rs
@@ -20,6 +20,13 @@ pub struct RustQuad {
pub bottom_left: RustPoint,
}
+#[derive(Clone, Copy, Debug)]
+pub struct RustCaptureRegion {
+ pub normalized_quad: RustQuad,
+ pub frame_width: u32,
+ pub frame_height: u32,
+}
+
#[derive(Clone, Copy, Debug)]
pub enum RustColorMode {
Color,
@@ -49,6 +56,7 @@ pub struct RustReprocessOptions {
pub struct RustScanResult {
// Decoded-source coordinates (EXIF applied, before `rotation_degrees`); `None` when nothing was detected.
pub quad: Option<RustQuad>,
+ pub needs_crop_review: bool,
pub color_mode: RustColorMode,
pub output_width: u32,
pub output_height: u32,
@@ -122,9 +130,18 @@ impl ScannerSession {
&self,
image_bytes: Vec<u8>,
max_pixels: Option<u32>,
+ region: Option<RustCaptureRegion>,
) -> Result<RustScanResult, RustScanError> {
catch_panic(|| {
- let result = self.inner.process_capture(&image_bytes, max_pixels)?;
+ let result = self.inner.process_capture_with_region(
+ &image_bytes,
+ max_pixels,
+ region.map(|region| scan::CaptureRegion {
+ normalized_quad: to_quad(region.normalized_quad),
+ frame_width: region.frame_width,
+ frame_height: region.frame_height,
+ }),
+ )?;
Ok(to_api_scan_result(result))
})
}
@@ -237,6 +254,7 @@ fn to_api_point(point: scan::Point) -> RustPoint {
fn to_api_scan_result(result: scan::ScanResult) -> RustScanResult {
RustScanResult {
quad: result.quad.map(to_api_quad),
+ needs_crop_review: result.needs_crop_review,
color_mode: to_api_color_mode(result.color_mode),
output_width: result.output_width,
output_height: result.output_height,
diff --git a/rust/crates/ml/src/scan/boundary.rs b/rust/crates/ml/src/scan/boundary.rs
index def09f1e07..de61d09044 100644
--- a/rust/crates/ml/src/scan/boundary.rs
+++ b/rust/crates/ml/src/scan/boundary.rs
@@ -147,7 +147,7 @@ fn simplify(mut hull: Vec<Point>, count: usize) -> Vec<Point> {
hull
}
-fn canonical(mut p: [Point; 4], extent: SourceExtent) -> Quad {
+pub(super) fn canonical(mut p: [Point; 4], extent: SourceExtent) -> Quad {
if area(&p) < 0.0 {
p.reverse();
}
@@ -942,6 +942,19 @@ pub(super) fn refine_capture(
source: &ImageU8,
quad: Quad,
needs_complete_support: bool,
+) -> OpResult<Option<Quad>> {
+ refine_capture_with_support(source, quad, needs_complete_support, 3)
+}
+
+pub(super) fn refine_capture_region(source: &ImageU8, quad: Quad) -> OpResult<Option<Quad>> {
+ refine_capture_with_support(source, quad, false, 4)
+}
+
+fn refine_capture_with_support(
+ source: &ImageU8,
+ quad: Quad,
+ needs_complete_support: bool,
+ minimum_supported_sides: usize,
) -> OpResult<Option<Quad>> {
let extent = SourceExtent::new(source.width, source.height)?;
validate_quad(quad, extent)?;
@@ -1051,7 +1064,7 @@ pub(super) fn refine_capture(
let color_samples = std::array::from_fn::<_, 4, _>(|side| {
color_edge_scores(&preview, [p[side], p[(side + 1) % 4]], band * 3.0)
});
- if (supported < 3 || needs_complete_support)
+ if (supported < minimum_supported_sides || needs_complete_support)
&& color_samples
.iter()
.any(|samples| samples.supported_positions() < 32)
diff --git a/rust/crates/ml/src/scan/mod.rs b/rust/crates/ml/src/scan/mod.rs
index 419bc08408..6a9636df81 100644
--- a/rust/crates/ml/src/scan/mod.rs
+++ b/rust/crates/ml/src/scan/mod.rs
@@ -25,7 +25,7 @@ pub struct Quad {
pub bottom_right: Point,
pub bottom_left: Point,
}
-pub use scanner::{ReprocessOptions, ScanError, ScanResult, ScannerSession};
+pub use scanner::{CaptureRegion, ReprocessOptions, ScanError, ScanResult, ScannerSession};
pub use segmentation::MASK_SIDE;
pub use yuv::PlaneLayout;
diff --git a/rust/crates/ml/src/scan/scanner.rs b/rust/crates/ml/src/scan/scanner.rs
index 5d1079d829..b26df9f0cf 100644
--- a/rust/crates/ml/src/scan/scanner.rs
+++ b/rust/crates/ml/src/scan/scanner.rs
@@ -1,17 +1,73 @@
use thiserror::Error;
use super::appearance::render_document;
-use super::boundary::{SearchBudget, locate, refine_capture};
+use super::boundary::{SearchBudget, canonical, locate, refine_capture, refine_capture_region};
use super::codec;
-use super::page::{RenderPlan, SourceExtent, quarter_turns, rotate_mask_quad};
+use super::page::{
+ RenderPlan, SourceExtent, points, quarter_turns, rotate_mask_quad, validate_quad,
+};
use super::segmentation::{MASK_SIDE, ProbabilityMap, Segmenter};
use super::yuv::{PlaneLayout, bgra_to_bgr, yuv420_to_bgr};
-use super::{ColorMode, Quad};
+use super::{ColorMode, Point, Quad};
use crate::cv::image::ImageU8;
const DEFAULT_MAX_PIXELS: u32 = 2_000_000;
const DEFAULT_JPEG_QUALITY: u8 = 75;
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct CaptureRegion {
+ pub normalized_quad: Quad,
+ pub frame_width: u32,
+ pub frame_height: u32,
+}
+
+impl CaptureRegion {
+ fn in_source(self, source: SourceExtent) -> Result<Quad, ScanError> {
+ let frame = SourceExtent::new(to_i32(self.frame_width)?, to_i32(self.frame_height)?)
+ .map_err(ScanError::InvalidInput)?;
+ validate_quad(
+ self.normalized_quad,
+ SourceExtent::new(1, 1).map_err(ScanError::InvalidInput)?,
+ )
+ .map_err(ScanError::InvalidInput)?;
+ let sx = source.width / frame.width;
+ let sy = source.height / frame.height;
+ let scale = sx.min(sy);
+ let width = frame.width * scale;
+ let height = frame.height * scale;
+ let mut corners = points(self.normalized_quad);
+ let tolerance = 1.5 / MASK_SIDE as f64;
+ for side in 0..4 {
+ let next = (side + 1) % 4;
+ for edge in [0.0, 1.0] {
+ if sx <= sy
+ && (corners[side].x - edge).abs() <= tolerance
+ && (corners[next].x - edge).abs() <= tolerance
+ {
+ corners[side].x = edge;
+ corners[next].x = edge;
+ }
+ if sy <= sx
+ && (corners[side].y - edge).abs() <= tolerance
+ && (corners[next].y - edge).abs() <= tolerance
+ {
+ corners[side].y = edge;
+ corners[next].y = edge;
+ }
+ }
+ }
+ let quad = canonical(
+ corners.map(|p| Point {
+ x: (source.width - width) / 2.0 + p.x * width,
+ y: (source.height - height) / 2.0 + p.y * height,
+ }),
+ source,
+ );
+ validate_quad(quad, source).map_err(ScanError::InvalidInput)?;
+ Ok(quad)
+ }
+}
+
#[derive(Debug, Clone, PartialEq, Eq, Error)]
pub enum ScanError {
#[error("invalid input: {0}")]
@@ -35,6 +91,7 @@ pub struct ReprocessOptions {
#[derive(Debug, Clone, PartialEq)]
pub struct ScanResult {
pub quad: Option<Quad>,
+ pub needs_crop_review: bool,
pub color_mode: ColorMode,
pub output_width: u32,
pub output_height: u32,
@@ -131,9 +188,18 @@ impl ScannerSession {
&self,
image_bytes: &[u8],
max_pixels: Option<u32>,
+ ) -> Result<ScanResult, ScanError> {
+ self.process_capture_with_region(image_bytes, max_pixels, None)
+ }
+
+ pub fn process_capture_with_region(
+ &self,
+ image_bytes: &[u8],
+ max_pixels: Option<u32>,
+ region: Option<CaptureRegion>,
) -> Result<ScanResult, ScanError> {
let start = std::time::Instant::now();
- let result = self.process_capture_inner(image_bytes, max_pixels);
+ let result = self.process_capture_inner(image_bytes, max_pixels, region);
match &result {
Ok(scan) => log::info!(
"capture: {}x{} -> {}x{} {:?}, quad {}, {}ms",
@@ -154,49 +220,58 @@ impl ScannerSession {
&self,
image_bytes: &[u8],
max_pixels: Option<u32>,
+ region: Option<CaptureRegion>,
) -> Result<ScanResult, ScanError> {
let budget = max_pixels.unwrap_or(DEFAULT_MAX_PIXELS);
validate_budget(budget)?;
let bgr = codec::decode_bgr(image_bytes)?;
let extent = SourceExtent::new(bgr.width, bgr.height).map_err(ScanError::InvalidInput)?;
- let mask = self.segment(&bgr)?;
- let detection = locate(&mask, extent, SearchBudget::Capture);
- log::debug!(
- "capture detection: {}, evidence {:.3}",
- detection.reason,
- detection.confidence
- );
- let quad = match detection.quad {
- Some(quad) => refine_capture(
- &bgr,
- quad.in_source(extent),
- quad.needs_complete_source_support,
- )
- .map_err(ScanError::Pipeline)?,
- None => None,
+ let quad = if let Some(region) = region {
+ refine_capture_region(&bgr, region.in_source(extent)?).map_err(ScanError::Pipeline)?
+ } else {
+ let mask = self.segment(&bgr)?;
+ let detection = locate(&mask, extent, SearchBudget::Capture);
+ log::debug!(
+ "capture detection: {}, evidence {:.3}",
+ detection.reason,
+ detection.confidence
+ );
+ match detection.quad {
+ Some(quad) => refine_capture(
+ &bgr,
+ quad.in_source(extent),
+ quad.needs_complete_source_support,
+ )
+ .map_err(ScanError::Pipeline)?,
+ None => None,
+ }
};
- let Some(quad) = quad else {
+ let mut result = if let Some(quad) = quad {
+ let plan = RenderPlan::new(quad, extent, 0, budget).map_err(ScanError::Pipeline)?;
+ let mut page = plan.render(&bgr).map_err(ScanError::Pipeline)?;
+ drop(bgr);
+ let mode = render_document(&mut page.image, page.valid.as_deref(), None)
+ .map_err(ScanError::Pipeline)?;
+ finish(Some(quad), mode, extent, page.image, DEFAULT_JPEG_QUALITY)?
+ } else {
let plan = RenderPlan::new(extent.full_frame(), extent, 0, budget)
.map_err(ScanError::Pipeline)?;
if plan.width == bgr.width && plan.height == bgr.height {
- return finish(None, ColorMode::Color, extent, bgr, DEFAULT_JPEG_QUALITY);
+ finish(None, ColorMode::Color, extent, bgr, DEFAULT_JPEG_QUALITY)?
+ } else {
+ let page = plan.render(&bgr).map_err(ScanError::Pipeline)?;
+ drop(bgr);
+ finish(
+ None,
+ ColorMode::Color,
+ extent,
+ page.image,
+ DEFAULT_JPEG_QUALITY,
+ )?
}
- let page = plan.render(&bgr).map_err(ScanError::Pipeline)?;
- drop(bgr);
- return finish(
- None,
- ColorMode::Color,
- extent,
- page.image,
- DEFAULT_JPEG_QUALITY,
- );
};
- let plan = RenderPlan::new(quad, extent, 0, budget).map_err(ScanError::Pipeline)?;
- let mut page = plan.render(&bgr).map_err(ScanError::Pipeline)?;
- drop(bgr);
- let mode = render_document(&mut page.image, page.valid.as_deref(), None)
- .map_err(ScanError::Pipeline)?;
- finish(Some(quad), mode, extent, page.image, DEFAULT_JPEG_QUALITY)
+ result.needs_crop_review = region.is_some() && quad.is_none();
+ Ok(result)
}
pub fn reprocess(
@@ -282,6 +357,7 @@ fn finish(
Ok(ScanResult {
quad,
+ needs_crop_review: false,
color_mode,
output_width,
output_height,
@@ -294,6 +370,320 @@ fn finish(
#[cfg(test)]
mod tests {
use super::*;
+ use crate::scan::page::quad_from_points;
+
+ fn document_source() -> Result<ImageU8, ScanError> {
+ let mut data = vec![30; 120 * 80 * 3];
+ for y in 20..60 {
+ for x in 30..90 {
+ for channel in 0..3 {
+ data[(y * 120 + x) * 3 + channel] = 210 + ((x + y + channel) % 10) as u8;
+ }
+ }
+ }
+ ImageU8::new(120, 80, 3, data).map_err(ScanError::Codec)
+ }
+
+ fn centered_region(width: u32, height: u32) -> CaptureRegion {
+ CaptureRegion {
+ normalized_quad: quad_from_points([
+ Point { x: 0.25, y: 0.25 },
+ Point { x: 0.75, y: 0.25 },
+ Point { x: 0.75, y: 0.75 },
+ Point { x: 0.25, y: 0.75 },
+ ]),
+ frame_width: width,
+ frame_height: height,
+ }
+ }
+
+ #[test]
+ fn capture_region_is_rendered_without_redetection_and_survives_reprocessing()
+ -> Result<(), ScanError> {
+ let session = ScannerSession {
+ segmenter: Segmenter::unavailable_for_render_tests(),
+ };
+ let source = document_source()?;
+ let bytes = codec::encode_jpeg(source, 90)?;
+ let region = centered_region(120, 80);
+ let result = session.process_capture_with_region(&bytes, Some(4_000_000), Some(region))?;
+ let expected_quad = result.quad.expect("supported page must be cropped");
+ assert!(!result.needs_crop_review);
+ assert!((result.output_width as i32 - 60).abs() <= 2);
+ assert!((result.output_height as i32 - 40).abs() <= 2);
+ let decoded = codec::decode_bgr(&result.processed_image)?;
+ assert_eq!(
+ (decoded.width as u32, decoded.height as u32),
+ (result.output_width, result.output_height)
+ );
+ let rotated = session.reprocess(
+ &bytes,
+ &ReprocessOptions {
+ quad: expected_quad,
+ rotation_degrees: 90,
+ color_mode: result.color_mode,
+ max_pixels: Some(4_000_000),
+ },
+ )?;
+ assert_eq!(rotated.quad, result.quad);
+ assert_eq!(
+ (rotated.output_width, rotated.output_height),
+ (result.output_height, result.output_width)
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn unsupported_capture_region_keeps_source_until_explicit_reprocessing() -> Result<(), ScanError>
+ {
+ let session = ScannerSession {
+ segmenter: Segmenter::unavailable_for_render_tests(),
+ };
+ let source = ImageU8::new(120, 80, 3, vec![180; 120 * 80 * 3]).map_err(ScanError::Codec)?;
+ let bytes = codec::encode_jpeg(source, 90)?;
+ let result =
+ session.process_capture_with_region(&bytes, None, Some(centered_region(120, 80)))?;
+ assert!(result.needs_crop_review);
+ assert_eq!(result.quad, None);
+ assert_eq!((result.output_width, result.output_height), (120, 80));
+ let confirmed = session.reprocess(
+ &bytes,
+ &ReprocessOptions {
+ quad: centered_region(120, 80)
+ .in_source(SourceExtent::new(120, 80).map_err(ScanError::InvalidInput)?)?,
+ rotation_degrees: 0,
+ color_mode: result.color_mode,
+ max_pixels: None,
+ },
+ )?;
+ assert!(!confirmed.needs_crop_review);
+ assert_eq!((confirmed.output_width, confirmed.output_height), (60, 40));
+ Ok(())
+ }
+
+ #[test]
+ fn capture_region_maps_centered_preview_to_different_still_aspects() -> Result<(), ScanError> {
+ for (frame, source, expected) in [
+ ((300, 400), (1200, 1600), (300.0, 400.0, 900.0, 1200.0)),
+ ((900, 1600), (1200, 1600), (375.0, 400.0, 825.0, 1200.0)),
+ ((400, 300), (1600, 900), (500.0, 225.0, 1100.0, 675.0)),
+ ] {
+ let region = centered_region(frame.0, frame.1);
+ let quad = region.in_source(
+ SourceExtent::new(source.0, source.1).map_err(ScanError::InvalidInput)?,
+ )?;
+ assert_eq!(
+ quad.top_left,
+ Point {
+ x: expected.0,
+ y: expected.1
+ }
+ );
+ assert_eq!(
+ quad.bottom_right,
+ Point {
+ x: expected.2,
+ y: expected.3
+ }
+ );
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn capture_region_preserves_output_across_cyclic_corner_orders() -> Result<(), ScanError> {
+ let session = ScannerSession {
+ segmenter: Segmenter::unavailable_for_render_tests(),
+ };
+ let bytes = codec::encode_jpeg(document_source()?, 90)?;
+ let region = centered_region(120, 80);
+ let expected = session.process_capture_with_region(&bytes, None, Some(region))?;
+ assert!(!expected.needs_crop_review);
+ let corners = points(region.normalized_quad);
+ for start in 1..4 {
+ let shifted = CaptureRegion {
+ normalized_quad: quad_from_points(std::array::from_fn(|i| {
+ corners[(i + start) % 4]
+ })),
+ ..region
+ };
+ assert_eq!(
+ session.process_capture_with_region(&bytes, None, Some(shifted))?,
+ expected
+ );
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn capture_region_distinguishes_photo_edges_from_interior_preview_edges()
+ -> Result<(), ScanError> {
+ let session = ScannerSession {
+ segmenter: Segmenter::unavailable_for_render_tests(),
+ };
+ for scale in [1, 10] {
+ for source_width in [110, 160] {
+ let width = source_width * scale;
+ let height = 80 * scale;
+ let mut data = vec![30; width * height * 3];
+ for y in 20 * scale..60 * scale {
+ for x in 70 * scale..150.min(source_width) * scale {
+ data[(y * width + x) * 3..(y * width + x + 1) * 3].fill(210);
+ }
+ }
+ let source =
+ ImageU8::new(width as i32, height as i32, 3, data).map_err(ScanError::Codec)?;
+ let bytes = codec::encode_jpeg(source, 95)?;
+ let wider = source_width == 160;
+ for edge in [0.996, 1.0, 1.004] {
+ let region = CaptureRegion {
+ normalized_quad: quad_from_points([
+ Point {
+ x: if wider { 1.0 / 3.0 } else { 70.0 / 110.0 },
+ y: 0.25,
+ },
+ Point { x: edge, y: 0.25 },
+ Point { x: edge, y: 0.75 },
+ Point {
+ x: if wider { 1.0 / 3.0 } else { 70.0 / 110.0 },
+ y: 0.75,
+ },
+ ]),
+ frame_width: if wider { 60 } else { 110 },
+ frame_height: 80,
+ };
+ let result = session.process_capture_with_region(&bytes, None, Some(region))?;
+ assert_eq!(
+ result.needs_crop_review, wider,
+ "{width}x{height}, edge {edge}"
+ );
+ if wider {
+ assert_eq!(result.quad, None);
+ assert_eq!(
+ (result.output_width, result.output_height),
+ (width as u32, height as u32)
+ );
+ } else {
+ let quad = result.quad.expect("photo edge must remain supported");
+ assert_eq!(quad.top_right.x, width as f64);
+ assert_eq!(quad.bottom_right.x, width as f64);
+ assert!((result.output_width as i32 - 40 * scale as i32).abs() <= 3);
+ }
+ }
+ }
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn invalid_capture_regions_fail_instead_of_silently_saving_a_full_photo()
+ -> Result<(), ScanError> {
+ let source = SourceExtent::new(120, 80).map_err(ScanError::InvalidInput)?;
+ let region = centered_region(120, 80);
+ for invalid in [
+ CaptureRegion {
+ frame_width: 0,
+ ..region
+ },
+ CaptureRegion {
+ frame_height: u32::MAX,
+ ..region
+ },
+ CaptureRegion {
+ normalized_quad: Quad {
+ top_left: Point {
+ x: f64::NAN,
+ y: 0.25,
+ },
+ ..region.normalized_quad
+ },
+ ..region
+ },
+ CaptureRegion {
+ normalized_quad: Quad {
+ top_left: region.normalized_quad.bottom_right,
+ ..region.normalized_quad
+ },
+ ..region
+ },
+ ] {
+ assert!(matches!(
+ invalid.in_source(source),
+ Err(ScanError::InvalidInput(_))
+ ));
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn capture_region_uses_exif_oriented_source_coordinates() -> Result<(), ScanError> {
+ let session = ScannerSession {
+ segmenter: Segmenter::unavailable_for_render_tests(),
+ };
+ let source = document_source()?;
+ let jpeg = codec::encode_jpeg(source, 90)?;
+ for (orientation, width, height) in [(1, 120, 80), (3, 120, 80), (6, 80, 120), (8, 80, 120)]
+ {
+ let exif = [
+ b'E',
+ b'x',
+ b'i',
+ b'f',
+ 0,
+ 0,
+ b'M',
+ b'M',
+ 0,
+ 42,
+ 0,
+ 0,
+ 0,
+ 8,
+ 0,
+ 1,
+ 1,
+ 18,
+ 0,
+ 3,
+ 0,
+ 0,
+ 0,
+ 1,
+ 0,
+ orientation,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ ];
+ let mut bytes = jpeg[..2].to_vec();
+ bytes.extend([0xff, 0xe1, 0, (exif.len() + 2) as u8]);
+ bytes.extend(exif);
+ bytes.extend(&jpeg[2..]);
+ let region = centered_region(width, height);
+ let capture = session.process_capture_with_region(&bytes, None, Some(region))?;
+ assert_eq!(
+ (capture.source_width, capture.source_height),
+ (width, height)
+ );
+ assert!(!capture.needs_crop_review);
+ assert!((capture.output_width as i32 - width as i32 / 2).abs() <= 2);
+ assert!((capture.output_height as i32 - height as i32 / 2).abs() <= 2);
+ let expected = session.reprocess(
+ &bytes,
+ &ReprocessOptions {
+ quad: capture.quad.expect("oriented page must be cropped"),
+ rotation_degrees: 0,
+ color_mode: capture.color_mode,
+ max_pixels: None,
+ },
+ )?;
+ assert_eq!(capture, expected);
+ }
+ Ok(())
+ }
#[test]
fn reprocess_needs_no_model_and_repeated_original_rotations_do_not_accumulate_loss()