Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Rebuild Locker document scanning pipeline

laurenspriem5ae8ccbcea3e
diff --git a/mobile/apps/locker/lib/services/scanner/auto_capture_controller.dart b/mobile/apps/locker/lib/services/scanner/auto_capture_controller.dart
index bcfe0027cd..d84b17dc28 100644
--- a/mobile/apps/locker/lib/services/scanner/auto_capture_controller.dart
+++ b/mobile/apps/locker/lib/services/scanner/auto_capture_controller.dart
@@ -1,4 +1,3 @@
-import 'package:locker/services/scanner/scan_geometry.dart';
import 'package:locker/services/scanner/scanner_models.dart';
enum AutoCaptureState { searching, arming, cooldown }
@@ -10,44 +9,57 @@ class AutoCaptureController {
static const clearHold = Duration(milliseconds: 700);
- static const maxFrameGap = Duration(milliseconds: 200);
-
static const minAreaFraction = 0.15;
AutoCaptureState _state = AutoCaptureState.searching;
double _progress = 0;
+ Duration _armedElapsed = Duration.zero;
Duration _graceElapsed = Duration.zero;
Duration _clearElapsed = Duration.zero;
- DateTime? _lastFrameAt;
+ Duration? _lastFrameAt;
AutoCaptureState get state => _state;
double get progress => _progress;
- bool onFrame(ScanQuad? stableQuad, {required bool captureBusy}) {
- final now = DateTime.now();
+ bool onFrame(
+ ScanQuad? stableQuad, {
+ required bool captureBusy,
+ required Duration timestamp,
+ bool resetProgress = false,
+ }) {
final last = _lastFrameAt;
- _lastFrameAt = now;
- var dt = last == null ? Duration.zero : now.difference(last);
- if (dt.isNegative) dt = Duration.zero;
- if (dt > maxFrameGap) dt = maxFrameGap;
+ if (timestamp.isNegative || (last != null && timestamp <= last)) {
+ invalidateArming();
+ return false;
+ }
+ _lastFrameAt = timestamp;
+ final dt = last == null || resetProgress ? Duration.zero : timestamp - last;
+ if (resetProgress) {
+ invalidateArming();
+ _clearElapsed = Duration.zero;
+ }
final eligible =
- stableQuad != null && _areaFraction(stableQuad) >= minAreaFraction;
+ !captureBusy &&
+ stableQuad != null &&
+ stableQuad.area >= minAreaFraction;
switch (_state) {
case AutoCaptureState.searching:
if (eligible) {
_state = AutoCaptureState.arming;
_progress = 0;
+ _armedElapsed = Duration.zero;
_graceElapsed = Duration.zero;
}
return false;
case AutoCaptureState.arming:
if (eligible) {
_graceElapsed = Duration.zero;
- _progress += dt.inMicroseconds / armHold.inMicroseconds;
- if (_progress >= 1) {
+ _armedElapsed += dt;
+ _progress = _armedElapsed.inMicroseconds / armHold.inMicroseconds;
+ if (_armedElapsed >= armHold) {
notifyCaptureStarted();
return true;
}
@@ -75,16 +87,24 @@ class AutoCaptureController {
void notifyCaptureStarted() {
_state = AutoCaptureState.cooldown;
_progress = 0;
+ _armedElapsed = Duration.zero;
_clearElapsed = Duration.zero;
}
+ void invalidateArming() {
+ if (_state != AutoCaptureState.arming) return;
+ _state = AutoCaptureState.searching;
+ _progress = 0;
+ _armedElapsed = Duration.zero;
+ _graceElapsed = Duration.zero;
+ }
+
void reset() {
_state = AutoCaptureState.searching;
_progress = 0;
+ _armedElapsed = Duration.zero;
_graceElapsed = Duration.zero;
_clearElapsed = Duration.zero;
_lastFrameAt = null;
}
-
- static double _areaFraction(ScanQuad quad) => quadArea(quad.corners);
}
diff --git a/mobile/apps/locker/lib/services/scanner/document_scanner_service.dart b/mobile/apps/locker/lib/services/scanner/document_scanner_service.dart
index dfeb2b114d..ca543a5165 100644
--- a/mobile/apps/locker/lib/services/scanner/document_scanner_service.dart
+++ b/mobile/apps/locker/lib/services/scanner/document_scanner_service.dart
@@ -226,23 +226,28 @@ class DocumentScannerService {
}
static ScanQuad _normalized(RustQuad quad, double width, double height) {
- Offset scale(RustPoint point) => Offset(point.x / width, point.y / height);
- return ScanQuad([
- scale(quad.topLeft),
- scale(quad.topRight),
- scale(quad.bottomRight),
- scale(quad.bottomLeft),
- ]);
+ return ScanQuad.fromSourcePixels([
+ for (final point in [
+ quad.topLeft,
+ quad.topRight,
+ quad.bottomRight,
+ quad.bottomLeft,
+ ])
+ Offset(point.x, point.y),
+ ], Size(width, height));
}
static RustQuad _inSourcePixels(ScanQuad quad, int width, int height) {
- RustPoint scale(Offset corner) =>
- RustPoint(x: corner.dx * width, y: corner.dy * height);
+ final corners = quad.toSourcePixels(
+ Size(width.toDouble(), height.toDouble()),
+ );
+ RustPoint at(int index) =>
+ RustPoint(x: corners[index].dx, y: corners[index].dy);
return RustQuad(
- topLeft: scale(quad.corners[0]),
- topRight: scale(quad.corners[1]),
- bottomRight: scale(quad.corners[2]),
- bottomLeft: scale(quad.corners[3]),
+ topLeft: at(0),
+ topRight: at(1),
+ bottomRight: at(2),
+ bottomLeft: at(3),
);
}
diff --git a/mobile/apps/locker/lib/services/scanner/pdf_writer.dart b/mobile/apps/locker/lib/services/scanner/pdf_writer.dart
index f8adda470b..d5ffcf4db2 100644
--- a/mobile/apps/locker/lib/services/scanner/pdf_writer.dart
+++ b/mobile/apps/locker/lib/services/scanner/pdf_writer.dart
@@ -1,4 +1,5 @@
import 'dart:convert';
+import 'dart:math' as math;
import 'dart:typed_data';
class JpegInfo {
@@ -58,39 +59,65 @@ class JpegInfo {
}
}
-class PdfPageSpec {
- const PdfPageSpec({
- required this.jpeg,
- required this.widthMm,
- required this.heightMm,
- });
+class PdfPageLayout {
+ const PdfPageLayout._({required this.widthMm, required this.heightMm});
+
+ static const maximumLongEdgeMm = 297.0;
+ static const maximumShortEdgeMm = 215.9;
+
+ factory PdfPageLayout.fromRaster({
+ required int widthPixels,
+ required int heightPixels,
+ }) {
+ if (widthPixels <= 0 || heightPixels <= 0) {
+ throw ArgumentError('Raster dimensions must be positive');
+ }
+ final landscape = widthPixels > heightPixels;
+ final shortToLong =
+ math.min(widthPixels, heightPixels) /
+ math.max(widthPixels, heightPixels);
+ final fitsLongEdge = maximumLongEdgeMm * shortToLong <= maximumShortEdgeMm;
+ final longEdgeMm = fitsLongEdge
+ ? maximumLongEdgeMm
+ : maximumShortEdgeMm / shortToLong;
+ final shortEdgeMm = fitsLongEdge
+ ? maximumLongEdgeMm * shortToLong
+ : maximumShortEdgeMm;
+ if (!shortEdgeMm.isFinite || shortEdgeMm <= 0) {
+ throw ArgumentError('Raster aspect ratio must be finite and positive');
+ }
+ return PdfPageLayout._(
+ widthMm: landscape ? longEdgeMm : shortEdgeMm,
+ heightMm: landscape ? shortEdgeMm : longEdgeMm,
+ );
+ }
- final Uint8List jpeg;
final double widthMm;
final double heightMm;
-}
-({double widthMm, double heightMm}) constrainToMaxFormat(
- double widthMm,
- double heightMm,
-) {
- const maxDim = 297.0;
- const minDim = 215.9;
- final scale = [
- maxDim / (widthMm > heightMm ? widthMm : heightMm),
- minDim / (widthMm < heightMm ? widthMm : heightMm),
- 1.0,
- ].reduce((a, b) => a < b ? a : b);
- return (widthMm: widthMm * scale, heightMm: heightMm * scale);
+ bool matchesRaster(JpegInfo info) {
+ if (info.width <= 0 || info.height <= 0) return false;
+ final ratio = widthMm / heightMm;
+ final rasterRatio = info.width / info.height;
+ return (ratio - rasterRatio).abs() <=
+ math.max(ratio, rasterRatio) * 32 * 2.220446049250313e-16;
+ }
}
-const double pointsPerMm = 72.0 / 25.4;
+class PdfPageSpec {
+ const PdfPageSpec({required this.jpeg, required this.layout});
+
+ final Uint8List jpeg;
+ final PdfPageLayout layout;
+}
class PdfWriter {
const PdfWriter({required this.creator});
final String creator;
+ static const _pointsPerMillimeter = 72.0 / 25.4;
+
Uint8List build(List<PdfPageSpec> pages, {DateTime? creationDate}) {
final out = BytesBuilder(copy: false);
final offsets = <int>[];
@@ -134,8 +161,11 @@ class PdfWriter {
for (var i = 0; i < pages.length; i++) {
final page = pages[i];
final info = JpegInfo.parse(page.jpeg);
- final widthPt = page.widthMm * pointsPerMm;
- final heightPt = page.heightMm * pointsPerMm;
+ if (!page.layout.matchesRaster(info)) {
+ throw ArgumentError('Page layout must preserve the JPEG aspect ratio');
+ }
+ final widthPt = page.layout.widthMm * _pointsPerMillimeter;
+ final heightPt = page.layout.heightMm * _pointsPerMillimeter;
startObject(pageObj(i));
text(
@@ -179,9 +209,7 @@ class PdfWriter {
}
static String _num(double value) {
- final rounded = (value * 1000).roundToDouble() / 1000;
- if (rounded == rounded.roundToDouble()) return rounded.toStringAsFixed(0);
- return rounded.toString();
+ return value.toStringAsFixed(9).replaceFirst(RegExp(r'\.?0+$'), '');
}
static String _escape(String value) => value
diff --git a/mobile/apps/locker/lib/services/scanner/quad_stability.dart b/mobile/apps/locker/lib/services/scanner/quad_stability.dart
new file mode 100644
index 0000000000..9a7b002778
--- /dev/null
+++ b/mobile/apps/locker/lib/services/scanner/quad_stability.dart
@@ -0,0 +1,96 @@
+import 'package:locker/services/scanner/scanner_models.dart';
+
+class QuadStabilitySample {
+ const QuadStabilitySample({
+ required this.quad,
+ required this.resetCapture,
+ this.validUntil,
+ });
+
+ final ScanQuad? quad;
+ final bool resetCapture;
+ final Duration? validUntil;
+}
+
+class QuadStability {
+ static const minimumFreshness = Duration(milliseconds: 600);
+ static const maximumFreshness = Duration(milliseconds: 1500);
+ static const maximumRelativeMotion = 0.04;
+
+ ScanQuad? _anchor;
+ ScanQuad? _previous;
+ Duration? _lastObservation;
+ Duration? _lastDetection;
+ int? _rotation;
+ Duration _freshness = minimumFreshness;
+
+ QuadStabilitySample update(
+ ScanQuad? detection, {
+ required Duration observedAt,
+ required Duration now,
+ required int rotationDegrees,
+ }) {
+ final age = now - observedAt;
+ final previousTime = _lastObservation;
+ final invalidTime =
+ age.isNegative ||
+ age >= maximumFreshness ||
+ observedAt.isNegative ||
+ (previousTime != null && observedAt <= previousTime);
+ if (invalidTime || rotationDegrees % 90 != 0) {
+ reset();
+ _lastObservation = previousTime;
+ return const QuadStabilitySample(quad: null, resetCapture: true);
+ }
+ final observationWindow = Duration(
+ microseconds: (age.inMicroseconds * 5 ~/ 2).clamp(
+ minimumFreshness.inMicroseconds,
+ maximumFreshness.inMicroseconds,
+ ),
+ );
+ final gapLimit = observationWindow > _freshness
+ ? observationWindow
+ : _freshness;
+ final rotation = rotationDegrees % 360;
+ final interrupted =
+ _rotation != null &&
+ (_rotation != rotation || observedAt - previousTime! > gapLimit);
+ if (interrupted) reset();
+ _freshness = observationWindow;
+ _lastObservation = observedAt;
+ _rotation = rotation;
+ if (detection == null) {
+ if (_lastDetection != null && observedAt - _lastDetection! >= gapLimit) {
+ _anchor = null;
+ _previous = null;
+ _lastDetection = null;
+ }
+ return QuadStabilitySample(quad: null, resetCapture: interrupted);
+ }
+ final previous = _previous;
+ final aligned = previous == null
+ ? detection
+ : detection.alignedTo(previous);
+ final anchor = _anchor;
+ final moved =
+ anchor != null &&
+ aligned.relativeMotionTo(anchor) > maximumRelativeMotion;
+ if (anchor == null || moved) _anchor = aligned;
+ _previous = aligned;
+ _lastDetection = observedAt;
+ return QuadStabilitySample(
+ quad: aligned,
+ resetCapture: interrupted || moved,
+ validUntil: observedAt + observationWindow,
+ );
+ }
+
+ void reset({Duration? observedBefore}) {
+ _anchor = null;
+ _previous = null;
+ _lastObservation = observedBefore;
+ _lastDetection = null;
+ _rotation = null;
+ _freshness = minimumFreshness;
+ }
+}
diff --git a/mobile/apps/locker/lib/services/scanner/scan_geometry.dart b/mobile/apps/locker/lib/services/scanner/scan_geometry.dart
deleted file mode 100644
index 3631c55d1b..0000000000
--- a/mobile/apps/locker/lib/services/scanner/scan_geometry.dart
+++ /dev/null
@@ -1,147 +0,0 @@
-import 'dart:math' as math;
-import 'dart:typed_data';
-import 'dart:ui';
-
-import 'package:locker/services/scanner/scanner_models.dart';
-
-ScanQuad lerpQuad(ScanQuad a, ScanQuad b, double t) => ScanQuad([
- for (var i = 0; i < 4; i++) Offset.lerp(a.corners[i], b.corners[i], t)!,
-]);
-
-double maxCornerDistance(ScanQuad a, ScanQuad b) {
- var worst = 0.0;
- for (var i = 0; i < 4; i++) {
- final d = (a.corners[i] - b.corners[i]).distance;
- if (d > worst) worst = d;
- }
- return worst;
-}
-
-double quadArea(List<Offset> corners) {
- var doubled = 0.0;
- for (var i = 0; i < corners.length; i++) {
- final a = corners[i];
- final b = corners[(i + 1) % corners.length];
- doubled += a.dx * b.dy - b.dx * a.dy;
- }
- return doubled.abs() / 2;
-}
-
-bool isUsableQuad(List<Offset> corners, {required double minAreaFraction}) {
- if (corners.length != 4) return false;
- final area = quadArea(corners);
- if (area < minAreaFraction) return false;
- final minCross = area * 0.02;
- var winding = 0.0;
- for (var i = 0; i < 4; i++) {
- final a = corners[i];
- final b = corners[(i + 1) % 4];
- final c = corners[(i + 2) % 4];
- final cross = (b.dx - a.dx) * (c.dy - b.dy) - (b.dy - a.dy) * (c.dx - b.dx);
- if (cross.abs() < minCross) return false;
- if (winding != 0 && cross.sign != winding) return false;
- winding = cross.sign;
- }
- return true;
-}
-
-ScanQuad orderClockwise(List<Offset> corners) {
- assert(corners.length == 4);
- final cx = corners.map((c) => c.dx).reduce((a, b) => a + b) / 4;
- final cy = corners.map((c) => c.dy).reduce((a, b) => a + b) / 4;
- final sorted = [...corners]
- ..sort(
- (a, b) => math
- .atan2(a.dy - cy, a.dx - cx)
- .compareTo(math.atan2(b.dy - cy, b.dx - cx)),
- );
- return ScanQuad(sorted);
-}
-
-Rect fittedRect(Size container, double aspect) {
- final containerAspect = container.width / container.height;
- final Size display = aspect > containerAspect
- ? Size(container.width, container.width / aspect)
- : Size(container.height * aspect, container.height);
- return Rect.fromLTWH(
- (container.width - display.width) / 2,
- (container.height - display.height) / 2,
- display.width,
- display.height,
- );
-}
-
-class QuadStabilizer {
- static const double maxCornerDrift = 20.0 / 256;
- static const int requiredStableFrames = 3;
-
- int _stableCount = 0;
- ScanQuad? _lastRawQuad;
-
- ScanQuad? update(ScanQuad? rawQuad) {
- final previous = _lastRawQuad;
- _lastRawQuad = rawQuad;
-
- if (rawQuad == null) {
- _stableCount = 0;
- return null;
- }
- if (previous == null) {
- _stableCount = 1;
- return null;
- }
-
- if (maxCornerDistance(previous, rawQuad) < maxCornerDrift) {
- _stableCount++;
- } else {
- _stableCount = 1;
- }
-
- return _stableCount >= requiredStableFrames ? rawQuad : null;
- }
-
- void reset() {
- _stableCount = 0;
- _lastRawQuad = null;
- }
-}
-
-Float64List? homographyMatrix(List<Offset> source, List<Offset> target) {
- assert(source.length == 4 && target.length == 4);
- final rows = List.generate(8, (_) => List.filled(9, 0.0));
- for (var i = 0; i < 4; i++) {
- final x = source[i].dx;
- final y = source[i].dy;
- final u = target[i].dx;
- final v = target[i].dy;
- rows[2 * i] = [x, y, 1, 0, 0, 0, -u * x, -u * y, u];
- rows[2 * i + 1] = [0, 0, 0, x, y, 1, -v * x, -v * y, v];
- }
- for (var col = 0; col < 8; col++) {
- var pivot = col;
- for (var r = col + 1; r < 8; r++) {
- if (rows[r][col].abs() > rows[pivot][col].abs()) pivot = r;
- }
- if (rows[pivot][col].abs() < 1e-9) return null;
- if (pivot != col) {
- final swap = rows[pivot];
- rows[pivot] = rows[col];
- rows[col] = swap;
- }
- for (var r = 0; r < 8; r++) {
- if (r == col) continue;
- final factor = rows[r][col] / rows[col][col];
- if (factor == 0) continue;
- for (var c = col; c < 9; c++) {
- rows[r][c] -= factor * rows[col][c];
- }
- }
- }
- final h = [for (var i = 0; i < 8; i++) rows[i][8] / rows[i][i], 1.0];
- return Float64List.fromList([
- h[0], h[3], 0, h[6], //
- h[1], h[4], 0, h[7], //
- 0, 0, 1, 0, //
- h[2], h[5], 0, h[8], //
- ]);
-}
diff --git a/mobile/apps/locker/lib/services/scanner/scan_session_controller.dart b/mobile/apps/locker/lib/services/scanner/scan_session_controller.dart
index 26aa86430d..ffd27311a1 100644
--- a/mobile/apps/locker/lib/services/scanner/scan_session_controller.dart
+++ b/mobile/apps/locker/lib/services/scanner/scan_session_controller.dart
@@ -169,9 +169,14 @@ class ScanSessionController extends ChangeNotifier {
final specs = <PdfPageSpec>[];
for (final page in List.of(_pages)) {
final jpeg = await page.processedJpeg.readAsBytes();
- final size = _pageSizeMm(page);
specs.add(
- PdfPageSpec(jpeg: jpeg, widthMm: size.widthMm, heightMm: size.heightMm),
+ PdfPageSpec(
+ jpeg: jpeg,
+ layout: PdfPageLayout.fromRaster(
+ widthPixels: page.width,
+ heightPixels: page.height,
+ ),
+ ),
);
}
final bytes = const PdfWriter(creator: 'Ente Locker').build(specs);
@@ -185,13 +190,6 @@ class ScanSessionController extends ChangeNotifier {
return file;
}
- static ({double widthMm, double heightMm}) _pageSizeMm(ScannedPage page) {
- const a4HeightMm = 297.0;
- final longestPx = page.width > page.height ? page.width : page.height;
- final scale = a4HeightMm / longestPx;
- return constrainToMaxFormat(page.width * scale, page.height * scale);
- }
-
Future<void> disposeSession() async {
if (_disposed) return;
_disposed = true;
diff --git a/mobile/apps/locker/lib/services/scanner/scanner_models.dart b/mobile/apps/locker/lib/services/scanner/scanner_models.dart
index d9a47bcb68..b23bbd7ada 100644
--- a/mobile/apps/locker/lib/services/scanner/scanner_models.dart
+++ b/mobile/apps/locker/lib/services/scanner/scanner_models.dart
@@ -1,15 +1,170 @@
import 'dart:io';
+import 'dart:math' as math;
import 'dart:ui';
enum ScanColorMode { color, grayscale }
class ScanQuad {
- const ScanQuad(this.corners);
+ ScanQuad._(List<Offset> corners) : corners = List.unmodifiable(corners);
+
+ factory ScanQuad(List<Offset> corners) {
+ final quad = tryFromOrdered(corners);
+ if (quad == null) {
+ throw ArgumentError.value(corners, 'corners', 'Expected a convex quad');
+ }
+ return quad;
+ }
final List<Offset> corners;
- factory ScanQuad.fullFrame() =>
- const ScanQuad([Offset(0, 0), Offset(1, 0), Offset(1, 1), Offset(0, 1)]);
+ factory ScanQuad.fullFrame() => ScanQuad._(const [
+ Offset(0, 0),
+ Offset(1, 0),
+ Offset(1, 1),
+ Offset(0, 1),
+ ]);
+
+ static ScanQuad? tryFromOrdered(List<Offset> corners) {
+ if (corners.length != 4 || corners.any((point) => !point.isFinite)) {
+ return null;
+ }
+ var extent = 0.0;
+ for (final a in corners) {
+ for (final b in corners) {
+ extent = math.max(extent, (a - b).distanceSquared);
+ }
+ }
+ if (!extent.isFinite || extent <= 0) return null;
+ final tolerance = extent * 128 * 2.220446049250313e-16;
+ for (var i = 0; i < 4; i++) {
+ final a = corners[(i + 1) % 4] - corners[i];
+ final b = corners[(i + 2) % 4] - corners[(i + 1) % 4];
+ final cross = a.dx * b.dy - a.dy * b.dx;
+ if (!cross.isFinite || cross <= tolerance) return null;
+ }
+ final quad = ScanQuad._(corners);
+ return quad.area.isFinite && quad.area > tolerance ? quad : null;
+ }
+
+ static ScanQuad? tryFromUnordered(List<Offset> corners) {
+ if (corners.length != 4 || corners.any((point) => !point.isFinite)) {
+ return null;
+ }
+ final center = corners.reduce((a, b) => a + b) / 4;
+ if (!center.isFinite) return null;
+ final sorted = [...corners]
+ ..sort((a, b) {
+ final aAngle = math.atan2(a.dy - center.dy, a.dx - center.dx);
+ final bAngle = math.atan2(b.dy - center.dy, b.dx - center.dx);
+ return aAngle.compareTo(bAngle);
+ });
+ var start = 0;
+ for (var i = 1; i < 4; i++) {
+ final candidate = sorted[i];
+ final current = sorted[start];
+ final score = (candidate.dx + candidate.dy).compareTo(
+ current.dx + current.dy,
+ );
+ if (score < 0 ||
+ (score == 0 &&
+ (candidate.dx < current.dx ||
+ (candidate.dx == current.dx && candidate.dy < current.dy)))) {
+ start = i;
+ }
+ }
+ return tryFromOrdered([
+ for (var i = 0; i < 4; i++) sorted[(start + i) % 4],
+ ]);
+ }
+
+ factory ScanQuad.fromSourcePixels(List<Offset> corners, Size sourceSize) {
+ _validateSize(sourceSize);
+ return ScanQuad([
+ for (final corner in corners)
+ Offset(corner.dx / sourceSize.width, corner.dy / sourceSize.height),
+ ]);
+ }
+
+ List<Offset> toSourcePixels(Size sourceSize) {
+ _validateSize(sourceSize);
+ final result = [
+ for (final corner in corners)
+ Offset(corner.dx * sourceSize.width, corner.dy * sourceSize.height),
+ ];
+ if (result.any((point) => !point.isFinite)) {
+ throw ArgumentError.value(sourceSize, 'sourceSize');
+ }
+ return result;
+ }
+
+ double get area {
+ final a = corners[1] - corners[0];
+ final b = corners[2] - corners[0];
+ final c = corners[3] - corners[0];
+ return (a.dx * b.dy - a.dy * b.dx + b.dx * c.dy - b.dy * c.dx) / 2;
+ }
+
+ double maxCornerDistanceTo(ScanQuad other) {
+ var distance = 0.0;
+ for (var i = 0; i < 4; i++) {
+ distance = math.max(distance, (corners[i] - other.corners[i]).distance);
+ }
+ return distance;
+ }
+
+ double relativeMotionTo(ScanQuad other) =>
+ maxCornerDistanceTo(other) / math.sqrt(math.min(area, other.area));
+
+ ScanQuad alignedTo(ScanQuad reference) {
+ var best = this;
+ var distance = maxCornerDistanceTo(reference);
+ for (var shift = 1; shift < 4; shift++) {
+ final candidate = ScanQuad._([
+ for (var i = 0; i < 4; i++) corners[(i + shift) % 4],
+ ]);
+ final candidateDistance = candidate.maxCornerDistanceTo(reference);
+ if (candidateDistance < distance) {
+ best = candidate;
+ distance = candidateDistance;
+ }
+ }
+ return best;
+ }
+
+ ScanQuad interpolateTo(ScanQuad other, double fraction) {
+ if (!fraction.isFinite || fraction < 0 || fraction > 1) {
+ throw ArgumentError.value(fraction, 'fraction');
+ }
+ final aligned = other.alignedTo(this);
+ return tryFromOrdered([
+ for (var i = 0; i < 4; i++)
+ Offset.lerp(corners[i], aligned.corners[i], fraction)!,
+ ]) ??
+ aligned;
+ }
+
+ ScanQuad rotatedClockwise(int degrees) {
+ if (degrees % 90 != 0) {
+ throw ArgumentError.value(degrees, 'degrees', 'Expected quarter turns');
+ }
+ var result = this;
+ for (var turn = 0; turn < (degrees ~/ 90) % 4; turn++) {
+ result = ScanQuad._([
+ for (var i = 0; i < 4; i++)
+ Offset(
+ 1 - result.corners[(i + 3) % 4].dy,
+ result.corners[(i + 3) % 4].dx,
+ ),
+ ]);
+ }
+ return result;
+ }
+
+ static void _validateSize(Size size) {
+ if (!size.width.isFinite || !size.height.isFinite || size.isEmpty) {
+ throw ArgumentError.value(size, 'sourceSize');
+ }
+ }
}
class ScannedPage {
diff --git a/mobile/apps/locker/lib/ui/pages/scanner/capture_flight.dart b/mobile/apps/locker/lib/ui/pages/scanner/capture_flight.dart
index be08e321c1..3a9f00862f 100644
--- a/mobile/apps/locker/lib/ui/pages/scanner/capture_flight.dart
+++ b/mobile/apps/locker/lib/ui/pages/scanner/capture_flight.dart
@@ -3,8 +3,8 @@ import 'dart:typed_data';
import 'dart:ui' as ui;
import 'package:flutter/material.dart';
-import 'package:locker/services/scanner/scan_geometry.dart';
import 'package:locker/services/scanner/scanner_models.dart';
+import 'package:locker/ui/pages/scanner/scan_projection.dart';
abstract final class CaptureFlightTuning {
static const arcHeight = 0.08;
@@ -324,7 +324,7 @@ void paintMappedSnapshot(
required double develop,
}) {
canvas.save();
- final homography = homographyMatrix(sourceQuad, imageQuad);
+ final homography = ScanProjection.projectiveTransform(sourceQuad, imageQuad);
if (homography != null) {
canvas.transform(homography);
} else {
diff --git a/mobile/apps/locker/lib/ui/pages/scanner/scan_projection.dart b/mobile/apps/locker/lib/ui/pages/scanner/scan_projection.dart
new file mode 100644
index 0000000000..16933289a1
--- /dev/null
+++ b/mobile/apps/locker/lib/ui/pages/scanner/scan_projection.dart
@@ -0,0 +1,156 @@
+import 'dart:math' as math;
+import 'dart:typed_data';
+import 'dart:ui';
+
+import 'package:locker/services/scanner/scanner_models.dart';
+
+class ScanProjection {
+ static Rect? fittedRect(Size viewport, double aspectRatio) {
+ if (!viewport.width.isFinite ||
+ !viewport.height.isFinite ||
+ viewport.isEmpty ||
+ !aspectRatio.isFinite ||
+ aspectRatio <= 0) {
+ return null;
+ }
+ final width = math.min(viewport.width, viewport.height * aspectRatio);
+ final height = width / aspectRatio;
+ if (!width.isFinite || !height.isFinite || width <= 0 || height <= 0) {
+ return null;
+ }
+ return Rect.fromLTWH(
+ (viewport.width - width) / 2,
+ (viewport.height - height) / 2,
+ width,
+ height,
+ );
+ }
+
+ static Float64List? projectiveTransform(
+ List<Offset> source,
+ List<Offset> destination,
+ ) {
+ final from = _Plane.tryCreate(source);
+ final to = _Plane.tryCreate(destination);
+ if (from == null || to == null) return null;
+ final equations = <List<double>>[];
+ for (var i = 0; i < 4; i++) {
+ final a = from.points[i];
+ final b = to.points[i];
+ equations
+ ..add([a.dx, a.dy, 1, 0, 0, 0, -b.dx * a.dx, -b.dx * a.dy, b.dx])
+ ..add([0, 0, 0, a.dx, a.dy, 1, -b.dy * a.dx, -b.dy * a.dy, b.dy]);
+ }
+ const tolerance = 4096 * 2.220446049250313e-16;
+ for (var column = 0; column < 8; column++) {
+ var pivot = column;
+ for (var row = column + 1; row < 8; row++) {
+ if (equations[row][column].abs() > equations[pivot][column].abs()) {
+ pivot = row;
+ }
+ }
+ final divisor = equations[pivot][column];
+ if (!divisor.isFinite || divisor.abs() <= tolerance) return null;
+ final swapped = equations[column];
+ equations[column] = equations[pivot];
+ equations[pivot] = swapped;
+ for (var j = column; j <= 8; j++) {
+ equations[column][j] /= divisor;
+ }
+ for (var row = 0; row < 8; row++) {
+ if (row == column) continue;
+ final factor = equations[row][column];
+ for (var j = column; j <= 8; j++) {
+ equations[row][j] -= factor * equations[column][j];
+ }
+ }
+ }
+ final normalized = [for (final row in equations) row[8], 1.0];
+ var sign = 0.0;
+ for (var i = 0; i < 4; i++) {
+ final point = from.points[i];
+ final denominator =
+ normalized[6] * point.dx + normalized[7] * point.dy + normalized[8];
+ if (!denominator.isFinite || denominator.abs() <= tolerance) return null;
+ if (sign != 0 && sign != denominator.sign) return null;
+ sign = denominator.sign;
+ final projected = Offset(
+ (normalized[0] * point.dx + normalized[1] * point.dy + normalized[2]) /
+ denominator,
+ (normalized[3] * point.dx + normalized[4] * point.dy + normalized[5]) /
+ denominator,
+ );
+ if (!projected.isFinite ||
+ (projected - to.points[i]).distance > math.sqrt(tolerance)) {
+ return null;
+ }
+ }
+ final matrix = _multiply(
+ [to.scale, 0, to.center.dx, 0, to.scale, to.center.dy, 0, 0, 1],
+ _multiply(normalized, [
+ 1 / from.scale,
+ 0,
+ -from.center.dx / from.scale,
+ 0,
+ 1 / from.scale,
+ -from.center.dy / from.scale,
+ 0,
+ 0,
+ 1,
+ ]),
+ );
+ if (matrix.any((value) => !value.isFinite)) return null;
+ return Float64List.fromList([
+ matrix[0],
+ matrix[3],
+ 0,
+ matrix[6],
+ matrix[1],
+ matrix[4],
+ 0,
+ matrix[7],
+ 0,
+ 0,
+ 1,
+ 0,
+ matrix[2],
+ matrix[5],
+ 0,
+ matrix[8],
+ ]);
+ }
+
+ static List<double> _multiply(List<double> a, List<double> b) => [
+ for (var row = 0; row < 3; row++)
+ for (var column = 0; column < 3; column++)
+ a[row * 3] * b[column] +
+ a[row * 3 + 1] * b[3 + column] +
+ a[row * 3 + 2] * b[6 + column],
+ ];
+}
+
+class _Plane {
+ const _Plane(this.points, this.center, this.scale);
+
+ final List<Offset> points;
+ final Offset center;
+ final double scale;
+
+ static _Plane? tryCreate(List<Offset> points) {
+ if (points.length != 4 || points.any((point) => !point.isFinite)) {
+ return null;
+ }
+ final center = points.reduce((a, b) => a + b) / 4;
+ var scale = 0.0;
+ for (final point in points) {
+ scale = math.max(scale, (point - center).distance);
+ }
+ if (!center.isFinite || !scale.isFinite || scale <= 0) return null;
+ final normalized = [for (final point in points) (point - center) / scale];
+ if (ScanQuad.tryFromOrdered(normalized) == null &&
+ ScanQuad.tryFromOrdered(normalized.reversed.toList()) == null) {
+ return null;
+ }
+ return _Plane(normalized, center, scale);
+ }
+}
diff --git a/mobile/apps/locker/lib/ui/pages/scanner/scan_quad_overlay.dart b/mobile/apps/locker/lib/ui/pages/scanner/scan_quad_overlay.dart
index 665b3fcc8a..7c1f001574 100644
--- a/mobile/apps/locker/lib/ui/pages/scanner/scan_quad_overlay.dart
+++ b/mobile/apps/locker/lib/ui/pages/scanner/scan_quad_overlay.dart
@@ -1,6 +1,5 @@
import 'package:flutter/material.dart';
import 'package:flutter/scheduler.dart' show Ticker;
-import 'package:locker/services/scanner/scan_geometry.dart';
import 'package:locker/services/scanner/scanner_models.dart';
class ScanQuadOverlay extends StatefulWidget {
@@ -51,9 +50,15 @@ class _ScanQuadOverlayState extends State<ScanQuadOverlay>
setState(() => _displayed = target);
return;
}
- if (maxCornerDistance(current, target) <= ScanQuadOverlay._settled) return;
+ final aligned = target.alignedTo(current);
+ if (current.maxCornerDistanceTo(aligned) <= ScanQuadOverlay._settled) {
+ return;
+ }
setState(
- () => _displayed = lerpQuad(current, target, ScanQuadOverlay._smoothing),
+ () => _displayed = current.interpolateTo(
+ aligned,
+ ScanQuadOverlay._smoothing,
+ ),
);
}
diff --git a/mobile/apps/locker/lib/ui/pages/scanner/scanner_capture_page.dart b/mobile/apps/locker/lib/ui/pages/scanner/scanner_capture_page.dart
index 43425c4a14..9f690f634c 100644
--- a/mobile/apps/locker/lib/ui/pages/scanner/scanner_capture_page.dart
+++ b/mobile/apps/locker/lib/ui/pages/scanner/scanner_capture_page.dart
@@ -11,7 +11,7 @@ import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:hugeicons/hugeicons.dart';
import 'package:locker/services/scanner/auto_capture_controller.dart';
-import 'package:locker/services/scanner/scan_geometry.dart';
+import 'package:locker/services/scanner/quad_stability.dart';
import 'package:locker/services/scanner/scan_session_controller.dart';
import 'package:locker/services/scanner/scanner_models.dart';
import 'package:locker/ui/pages/scanner/capture_flight.dart';
@@ -35,8 +35,11 @@ class _ScannerCapturePageState extends State<ScannerCapturePage>
with WidgetsBindingObserver {
final _logger = Logger('ScannerCapturePage');
final _session = ScanSessionController();
- final _stabilizer = QuadStabilizer();
+ final _stabilizer = QuadStability();
final _autoCapture = AutoCaptureController();
+ final _frameClock = Stopwatch()..start();
+ Timer? _quadExpiry;
+ int _analysisGeneration = 0;
CameraController? _camera;
_CameraStatus _status = _CameraStatus.starting;
@@ -86,6 +89,8 @@ class _ScannerCapturePageState extends State<ScannerCapturePage>
@override
void dispose() {
+ _quadExpiry?.cancel();
+ _frameClock.stop();
WidgetsBinding.instance.removeObserver(this);
_session.removeListener(_onSessionChanged);
unawaited(_camera?.dispose());
@@ -103,6 +108,7 @@ class _ScannerCapturePageState extends State<ScannerCapturePage>
void didChangeAppLifecycleState(AppLifecycleState state) {
final camera = _camera;
if (state == AppLifecycleState.inactive) {
+ _resetLiveTracking();
if (camera != null) {
setState(() => _camera = null);
unawaited(camera.dispose());
@@ -126,11 +132,11 @@ class _ScannerCapturePageState extends State<ScannerCapturePage>
}
Future<void> _startCamera() async {
+ _resetLiveTracking();
setState(() {
_status = _CameraStatus.starting;
_stableQuad = null;
});
- _stabilizer.reset();
_autoCapture.reset();
try {
final cameras = await availableCameras();
@@ -186,33 +192,42 @@ class _ScannerCapturePageState extends State<ScannerCapturePage>
}
Future<void> _pauseCamera() async {
+ _resetLiveTracking();
final camera = _camera;
if (camera == null) return;
setState(() {
_camera = null;
_stableQuad = null;
});
- _stabilizer.reset();
_autoCapture.reset();
await camera.dispose();
}
void _onFrame(CameraImage image) {
+ final camera = _camera;
if (_reviewActive ||
+ camera == null ||
_analysisInFlight ||
_takingPicture ||
!_session.isServiceReady) {
return;
}
_analysisInFlight = true;
- unawaited(_analyze(image));
+ unawaited(
+ _analyze(image, camera, _frameClock.elapsed, _analysisGeneration),
+ );
}
- Future<void> _analyze(CameraImage image) async {
+ Future<void> _analyze(
+ CameraImage image,
+ CameraController camera,
+ Duration observedAt,
+ int generation,
+ ) async {
try {
final rotation = Platform.isIOS
? 0
- : _camera?.description.sensorOrientation ?? 0;
+ : camera.description.sensorOrientation;
ScanQuad? raw;
if (image.planes.length >= 3) {
raw = await _session.detectLiveYuv(
@@ -235,24 +250,57 @@ class _ScannerCapturePageState extends State<ScannerCapturePage>
rotation,
);
}
- final stable = _stabilizer.update(raw);
+ if (!mounted ||
+ camera != _camera ||
+ generation != _analysisGeneration ||
+ _takingPicture) {
+ return;
+ }
+ final now = _frameClock.elapsed;
+ final sample = _stabilizer.update(
+ raw,
+ observedAt: observedAt,
+ now: now,
+ rotationDegrees: rotation,
+ );
+ _quadExpiry?.cancel();
+ if (sample.quad != null) {
+ _quadExpiry = Timer(sample.validUntil! - now, () {
+ if (!mounted) return;
+ _resetLiveTracking();
+ _autoCapture.invalidateArming();
+ setState(() {});
+ });
+ }
var fire = false;
if (_autoMode) {
fire = _autoCapture.onFrame(
- stable,
+ sample.quad,
captureBusy: _takingPicture || _session.isProcessing,
+ timestamp: observedAt,
+ resetProgress: sample.resetCapture,
);
}
- if (mounted) {
- setState(() => _stableQuad = stable);
- if (fire) unawaited(_capture());
- }
+ setState(() => _stableQuad = sample.quad);
+ if (fire) unawaited(_capture());
} catch (_) {
+ if (mounted && generation == _analysisGeneration && camera == _camera) {
+ _resetLiveTracking();
+ _autoCapture.invalidateArming();
+ setState(() {});
+ }
} finally {
_analysisInFlight = false;
}
}
+ void _resetLiveTracking() {
+ _analysisGeneration++;
+ _quadExpiry?.cancel();
+ _stabilizer.reset(observedBefore: _frameClock.elapsed);
+ _stableQuad = null;
+ }
+
Future<void> _capture() async {
final camera = _camera;
if (camera == null || _takingPicture || _reviewActive) return;
@@ -265,7 +313,7 @@ class _ScannerCapturePageState extends State<ScannerCapturePage>
_snapQuad = quad;
_snapId++;
});
- _stabilizer.reset();
+ _resetLiveTracking();
try {
final shot = await camera.takePicture();
final bytes = await shot.readAsBytes();
diff --git a/mobile/apps/locker/lib/ui/pages/scanner/scanner_crop_page.dart b/mobile/apps/locker/lib/ui/pages/scanner/scanner_crop_page.dart
index 2a1b86cf8f..ea6b6b3165 100644
--- a/mobile/apps/locker/lib/ui/pages/scanner/scanner_crop_page.dart
+++ b/mobile/apps/locker/lib/ui/pages/scanner/scanner_crop_page.dart
@@ -5,9 +5,9 @@ import 'package:ente_strings/ente_strings.dart';
import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:hugeicons/hugeicons.dart';
-import 'package:locker/services/scanner/scan_geometry.dart';
import 'package:locker/services/scanner/scan_session_controller.dart';
import 'package:locker/services/scanner/scanner_models.dart';
+import 'package:locker/ui/pages/scanner/scan_projection.dart';
class ScannerCropPage extends StatefulWidget {
const ScannerCropPage({
@@ -66,15 +66,14 @@ class _ScannerCropPageState extends State<ScannerCropPage> {
return null;
}
- ScanQuad get _orderedQuad => orderClockwise(_corners);
+ ScanQuad? get _orderedQuad => ScanQuad.tryFromUnordered(_corners);
- bool get _canSave =>
- _corners.length == 4 &&
- isUsableQuad(_orderedQuad.corners, minAreaFraction: _minAreaFraction);
+ bool get _canSave => (_orderedQuad?.area ?? 0) >= _minAreaFraction;
Future<void> _save() async {
final navigator = Navigator.of(context);
final quad = _orderedQuad;
+ if (quad == null || quad.area < _minAreaFraction) return;
await widget.session.updatePage(widget.pageId, quad: quad);
if (mounted) navigator.pop();
}
@@ -170,7 +169,11 @@ class _ScannerCropPageState extends State<ScannerCropPage> {
constraints.maxWidth,
constraints.maxHeight,
);
- final rect = fittedRect(container, _aspect);
+ final rect = ScanProjection.fittedRect(
+ container,
+ _aspect,
+ );
+ if (rect == null) return const SizedBox.shrink();
Offset toScreen(Offset normalized) => Offset(
rect.left + normalized.dx * rect.width,
diff --git a/mobile/apps/locker/lib/ui/pages/scanner/scanner_review_page.dart b/mobile/apps/locker/lib/ui/pages/scanner/scanner_review_page.dart
index 651c97d2fe..7821fbefe8 100644
--- a/mobile/apps/locker/lib/ui/pages/scanner/scanner_review_page.dart
+++ b/mobile/apps/locker/lib/ui/pages/scanner/scanner_review_page.dart
@@ -9,11 +9,11 @@ import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:hugeicons/hugeicons.dart';
import 'package:intl/intl.dart';
-import 'package:locker/services/scanner/scan_geometry.dart';
import 'package:locker/services/scanner/scan_session_controller.dart';
import 'package:locker/services/scanner/scanner_models.dart';
import 'package:locker/ui/components/text_input_sheet.dart';
import 'package:locker/ui/pages/scanner/delayed_reveal.dart';
+import 'package:locker/ui/pages/scanner/scan_projection.dart';
import 'package:locker/ui/pages/scanner/scanner_crop_page.dart';
import 'package:share_plus/share_plus.dart';
@@ -259,8 +259,9 @@ class _ScannerReviewPageState extends State<ScannerReviewPage>
return LayoutBuilder(
builder: (context, constraints) {
final box = constraints.biggest;
- final before = fittedRect(box, aspect);
- final after = fittedRect(box, 1 / aspect);
+ final before = ScanProjection.fittedRect(box, aspect);
+ final after = ScanProjection.fittedRect(box, 1 / aspect);
+ if (before == null || after == null) return const SizedBox.shrink();
final targetScale = after.width / before.height;
return AnimatedBuilder(
animation: _rotateController,
diff --git a/rust/crates/ml/examples/scan.rs b/rust/crates/ml/examples/scan.rs
new file mode 100644
index 0000000000..33245e3dd9
--- /dev/null
+++ b/rust/crates/ml/examples/scan.rs
@@ -0,0 +1,75 @@
+use std::time::Instant;
+
+use ente_ml::scan::{ColorMode, Point, Quad, ReprocessOptions, ScannerSession};
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+ let arguments: Vec<String> = std::env::args().collect();
+ if arguments.len() < 4 {
+ return Err("usage: scan MODEL INPUT OUTPUT [PIXELS] [ROTATION] [color|gray] [x0,y0,x1,y1,x2,y2,x3,y3]".into());
+ }
+ let budget = arguments
+ .get(4)
+ .map(|v| v.parse::<u32>())
+ .transpose()?
+ .unwrap_or(2_000_000);
+ let rotation = arguments
+ .get(5)
+ .map(|v| v.parse::<i32>())
+ .transpose()?
+ .unwrap_or(0);
+ let mode = if arguments.get(6).is_some_and(|s| s == "gray") {
+ ColorMode::Grayscale
+ } else {
+ ColorMode::Color
+ };
+ let start = Instant::now();
+ let session = ScannerSession::new(&arguments[1])?;
+ let load_ms = start.elapsed().as_secs_f64() * 1000.0;
+ let bytes = std::fs::read(&arguments[2])?;
+ let start = Instant::now();
+ let result = if let Some(coordinates) = arguments.get(7) {
+ let values = coordinates
+ .split(',')
+ .map(str::parse::<f64>)
+ .collect::<Result<Vec<_>, _>>()?;
+ if values.len() != 8 {
+ return Err("quad requires eight coordinates".into());
+ }
+ let point = |i| Point {
+ x: values[i],
+ y: values[i + 1],
+ };
+ let quad = Quad {
+ top_left: point(0),
+ top_right: point(2),
+ bottom_right: point(4),
+ bottom_left: point(6),
+ };
+ session.reprocess(
+ &bytes,
+ &ReprocessOptions {
+ quad,
+ rotation_degrees: rotation,
+ color_mode: mode,
+ max_pixels: Some(budget),
+ },
+ )?
+ } else {
+ session.process_capture(&bytes, Some(budget))?
+ };
+ let process_ms = start.elapsed().as_secs_f64() * 1000.0;
+ std::fs::write(&arguments[3], &result.processed_image)?;
+ let quad = result.quad.map(|q| {
+ vec![
+ [q.top_left.x, q.top_left.y],
+ [q.top_right.x, q.top_right.y],
+ [q.bottom_right.x, q.bottom_right.y],
+ [q.bottom_left.x, q.bottom_left.y],
+ ]
+ });
+ println!(
+ "{}",
+ serde_json::json!({"source":[result.source_width,result.source_height],"output":[result.output_width,result.output_height],"mode":format!("{:?}",result.color_mode),"quad":quad,"bytes":result.processed_image.len(),"load_ms":load_ms,"process_ms":process_ms})
+ );
+ Ok(())
+}
diff --git a/rust/crates/ml/src/cv/bilateral.rs b/rust/crates/ml/src/cv/bilateral.rs
deleted file mode 100644
index 64dead5a9b..0000000000
--- a/rust/crates/ml/src/cv/bilateral.rs
+++ /dev/null
@@ -1,88 +0,0 @@
-use rayon::prelude::*;
-
-use super::reflect101;
-use crate::cv::image::ImageU8;
-use crate::cv::{OpResult, PARALLEL_MIN_ELEMS};
-
-pub(crate) fn bilateral_filter_u8(
- src: &ImageU8,
- d: i32,
- sigma_color: f64,
- sigma_space: f64,
-) -> OpResult<ImageU8> {
- if src.channels != 1 {
- return Err("bilateral_filter_u8: expected a single-channel image".to_string());
- }
- if sigma_color <= 1e-6 || sigma_space <= 1e-6 {
- return Ok(src.clone());
- }
- let radius = if d <= 0 {
- (sigma_space * 1.5).round() as i32
- } else {
- d / 2
- }
- .max(1) as usize;
- let (w, h) = (src.width as usize, src.height as usize);
-
- let stride = w + 2 * radius;
- let mut temp = vec![0u8; stride * (h + 2 * radius)];
- let cols: Vec<usize> = (0..stride)
- .map(|x| reflect101(x as i64 - radius as i64, src.width) as usize)
- .collect();
- for (y, trow) in temp.chunks_exact_mut(stride).enumerate() {
- let sy = reflect101(y as i64 - radius as i64, src.height) as usize;
- let srow = &src.data[sy * w..(sy + 1) * w];
- for (t, &sx) in trow.iter_mut().zip(cols.iter()) {
- *t = srow[sx];
- }
- }
-
- let color_coeff = -0.5 / (sigma_color * sigma_color);
- let space_coeff = -0.5 / (sigma_space * sigma_space);
- let color_weight: Vec<f32> = (0..256)
- .map(|i| ((i * i) as f64 * color_coeff).exp() as f32)
- .collect();
-
- let ir = radius as i64;
- let mut taps = Vec::new();
- for i in -ir..=ir {
- for j in -ir..=ir {
- let r2 = (i * i + j * j) as f64;
- if r2 > (ir * ir) as f64 {
- continue;
- }
- taps.push((
- ((i + ir) as usize * stride + (j + ir) as usize),
- (r2 * space_coeff).exp() as f32,
- ));
- }
- }
-
- let mut out = vec![0u8; w * h];
- let row = |y: usize, dst: &mut [u8]| {
- let win = &temp[y * stride..];
- let centre = radius * stride + radius;
- for (x, d) in dst.iter_mut().enumerate() {
- let mid = win[centre + x] as i32;
- let mut sum = 0.0f32;
- let mut wsum = 0.0f32;
- for &(off, sw) in &taps {
- let val = win[off + x];
- let weight = sw * color_weight[(val as i32 - mid).unsigned_abs() as usize];
- wsum += weight;
- sum += val as f32 * weight;
- }
- *d = (sum / wsum).round_ties_even() as u8;
- }
- };
- if w * h >= PARALLEL_MIN_ELEMS {
- out.par_chunks_exact_mut(w)
- .enumerate()
- .for_each(|(y, dst)| row(y, dst));
- } else {
- for (y, dst) in out.chunks_exact_mut(w).enumerate() {
- row(y, dst);
- }
- }
- ImageU8::new(src.width, src.height, 1, out)
-}
diff --git a/rust/crates/ml/src/cv/canny.rs b/rust/crates/ml/src/cv/canny.rs
deleted file mode 100644
index 4a4d81c30e..0000000000
--- a/rust/crates/ml/src/cv/canny.rs
+++ /dev/null
@@ -1,151 +0,0 @@
-// The 75/200 thresholds used downstream are calibrated against this exact
-// gradient scaling: integer `|dx| + |dy|` of a 3x3 Sobel with replicated
-// borders, thresholds floored, the low one compared strictly.
-
-use crate::cv::OpResult;
-use crate::cv::image::ImageU8;
-
-const CANNY_SHIFT: i32 = 15;
-// `(int)(tan(22.5 deg) * (1 << CANNY_SHIFT) + 0.5)`.
-const TG22: i32 = 13573;
-
-const MAYBE: u8 = 0;
-const NO: u8 = 1;
-const EDGE: u8 = 2;
-
-fn sobel_planes(src: &[u8], w: usize, h: usize, cn: usize, c: usize) -> (Vec<i16>, Vec<i16>) {
- let mut h_diff = vec![0i16; w * h];
- let mut h_smooth = vec![0i16; w * h];
- for y in 0..h {
- let row = &src[y * w * cn..(y + 1) * w * cn];
- let at = |x: usize| row[x * cn + c] as i16;
- let (d, s) = (
- &mut h_diff[y * w..(y + 1) * w],
- &mut h_smooth[y * w..(y + 1) * w],
- );
- for x in 0..w {
- let left = at(x.saturating_sub(1));
- let right = at((x + 1).min(w - 1));
- d[x] = right - left;
- s[x] = left + 2 * at(x) + right;
- }
- }
-
- let mut dx = vec![0i16; w * h];
- let mut dy = vec![0i16; w * h];
- for y in 0..h {
- let up = y.saturating_sub(1) * w;
- let down = (y + 1).min(h - 1) * w;
- let here = y * w;
- for x in 0..w {
- dx[here + x] = h_diff[up + x] + 2 * h_diff[here + x] + h_diff[down + x];
- dy[here + x] = h_smooth[down + x] - h_smooth[up + x];
- }
- }
- (dx, dy)
-}
-
-fn floor_threshold(t: f64) -> i32 {
- t.floor().clamp(i32::MIN as f64, i32::MAX as f64) as i32
-}
-
-pub(crate) fn canny(src: &ImageU8, threshold1: f64, threshold2: f64) -> OpResult<ImageU8> {
- let (w, h) = (src.width as usize, src.height as usize);
- let cn = src.channels as usize;
-
- let (low_thresh, high_thresh) = if threshold1 > threshold2 {
- (threshold2, threshold1)
- } else {
- (threshold1, threshold2)
- };
- let low = floor_threshold(low_thresh);
- let high = floor_threshold(high_thresh);
-
- let (mut dx, mut dy) = sobel_planes(&src.data, w, h, cn, 0);
- let mut norm: Vec<i32> = dx
- .iter()
- .zip(dy.iter())
- .map(|(&x, &y)| (x as i32).abs() + (y as i32).abs())
- .collect();
- for c in 1..cn {
- let (cdx, cdy) = sobel_planes(&src.data, w, h, cn, c);
- for i in 0..w * h {
- let m = (cdx[i] as i32).abs() + (cdy[i] as i32).abs();
- if m > norm[i] {
- norm[i] = m;
- dx[i] = cdx[i];
- dy[i] = cdy[i];
- }
- }
- }
-
- let step = w + 2;
- let mut mag = vec![0i32; step * (h + 2)];
- for y in 0..h {
- mag[(y + 1) * step + 1..(y + 1) * step + 1 + w].copy_from_slice(&norm[y * w..(y + 1) * w]);
- }
- let mut map = vec![NO; step * (h + 2)];
- let mut stack: Vec<usize> = Vec::new();
-
- for y in 0..h {
- for x in 0..w {
- let p = (y + 1) * step + x + 1;
- let m = mag[p];
- if m > low {
- let (xs, ys) = (dx[y * w + x] as i32, dy[y * w + x] as i32);
- let ax = xs.abs();
- let ay = ys.abs() << CANNY_SHIFT;
- let tg22x = ax * TG22;
- let keep = if ay < tg22x {
- m > mag[p - 1] && m >= mag[p + 1]
- } else if ay > tg22x + (ax << (CANNY_SHIFT + 1)) {
- m > mag[p - step] && m >= mag[p + step]
- } else {
- let s: isize = if (xs ^ ys) < 0 { -1 } else { 1 };
- let up = (p as isize - step as isize - s) as usize;
- let down = (p as isize + step as isize + s) as usize;
- m > mag[up] && m > mag[down]
- };
- if keep {
- if m > high {
- map[p] = EDGE;
- stack.push(p);
- } else {
- map[p] = MAYBE;
- }
- continue;
- }
- }
- map[p] = NO;
- }
- }
-
- let offsets = [
- -(step as isize) - 1,
- -(step as isize),
- -(step as isize) + 1,
- -1,
- 1,
- step as isize - 1,
- step as isize,
- step as isize + 1,
- ];
- while let Some(p) = stack.pop() {
- for offset in offsets {
- let q = (p as isize + offset) as usize;
- if map[q] == MAYBE {
- map[q] = EDGE;
- stack.push(q);
- }
- }
- }
-
- let mut out = vec![0u8; w * h];
- for y in 0..h {
- let row = &map[(y + 1) * step + 1..(y + 1) * step + 1 + w];
- for (o, &v) in out[y * w..(y + 1) * w].iter_mut().zip(row.iter()) {
- *o = if v == EDGE { 255 } else { 0 };
- }
- }
- ImageU8::new(src.width, src.height, 1, out)
-}
diff --git a/rust/crates/ml/src/cv/channels.rs b/rust/crates/ml/src/cv/channels.rs
deleted file mode 100644
index 6f453d8b04..0000000000
--- a/rust/crates/ml/src/cv/channels.rs
+++ /dev/null
@@ -1,81 +0,0 @@
-use crate::cv::OpResult;
-use crate::cv::image::ImageU8;
-
-// 15-bit fixed-point BT.601 luma weights.
-const GRAY_SHIFT: i32 = 15;
-const RY15: i32 = 9798;
-const GY15: i32 = 19235;
-const BY15: i32 = 3735;
-
-fn require_channels(src: &ImageU8, channels: i32, op: &str) -> OpResult<()> {
- if src.channels != channels {
- return Err(format!(
- "{op}: expected {channels} channel(s), got {}",
- src.channels
- ));
- }
- Ok(())
-}
-
-pub(crate) fn gray_to_bgr(src: &ImageU8) -> OpResult<ImageU8> {
- require_channels(src, 1, "gray_to_bgr")?;
- let mut data = vec![0u8; src.data.len() * 3];
- super::pointwise(&mut data, 3, &src.data, 1, |data, srcd| {
- for (out, &v) in data.as_chunks_mut::<3>().0.iter_mut().zip(srcd.iter()) {
- out[0] = v;
- out[1] = v;
- out[2] = v;
- }
- });
- ImageU8::new(src.width, src.height, 3, data)
-}
-
-pub(crate) fn bgr_to_gray(src: &ImageU8) -> OpResult<ImageU8> {
- require_channels(src, 3, "bgr_to_gray")?;
- let round = 1i32 << (GRAY_SHIFT - 1);
- let mut data = vec![0u8; src.data.len() / 3];
- super::pointwise(&mut data, 1, &src.data, 3, |data, srcd| {
- for (out, px) in data.iter_mut().zip(srcd.as_chunks::<3>().0.iter()) {
- let sum = px[0] as i32 * BY15 + px[1] as i32 * GY15 + px[2] as i32 * RY15;
- *out = ((sum + round) >> GRAY_SHIFT) as u8;
- }
- });
- ImageU8::new(src.width, src.height, 1, data)
-}
-
-pub(crate) fn split_u8(src: &ImageU8) -> OpResult<Vec<ImageU8>> {
- let cn = src.channels as usize;
- let mut planes = Vec::with_capacity(cn);
- for c in 0..cn {
- let mut data = vec![0u8; src.data.len() / cn];
- super::pointwise(&mut data, 1, &src.data, cn, |data, srcd| {
- for (out, px) in data.iter_mut().zip(srcd.chunks_exact(cn)) {
- *out = px[c];
- }
- });
- planes.push(ImageU8::new(src.width, src.height, 1, data)?);
- }
- Ok(planes)
-}
-
-pub(crate) fn merge_u8(channels: &[ImageU8]) -> OpResult<ImageU8> {
- let first = channels
- .first()
- .ok_or_else(|| "merge_u8: no input planes".to_string())?;
- for plane in channels {
- if plane.channels != 1 {
- return Err("merge_u8: every input plane must be single-channel".to_string());
- }
- if plane.width != first.width || plane.height != first.height {
- return Err("merge_u8: input planes have different sizes".to_string());
- }
- }
- let cn = channels.len();
- let mut data = vec![0u8; first.data.len() * cn];
- for (c, plane) in channels.iter().enumerate() {
- for (i, &v) in plane.data.iter().enumerate() {
- data[i * cn + c] = v;
- }
- }
- ImageU8::new(first.width, first.height, cn as i32, data)
-}
diff --git a/rust/crates/ml/src/cv/draw.rs b/rust/crates/ml/src/cv/draw.rs
deleted file mode 100644
index 5a46988da7..0000000000
--- a/rust/crates/ml/src/cv/draw.rs
+++ /dev/null
@@ -1,47 +0,0 @@
-use image::{GrayImage, Luma};
-use imageproc::drawing::draw_polygon_mut;
-use imageproc::point::Point;
-
-use super::saturate_u8_f64;
-use crate::cv::OpResult;
-use crate::cv::image::ImageU8;
-
-pub(crate) fn fill_poly(
- width: i32,
- height: i32,
- polygon: &[(i32, i32)],
- value: f64,
-) -> OpResult<ImageU8> {
- if width <= 0 || height <= 0 {
- return Err(format!("fill_poly: invalid mask size {width}x{height}"));
- }
- let mut canvas = GrayImage::new(width as u32, height as u32);
- let color = Luma([saturate_u8_f64(value)]);
-
- let mut pts: Vec<Point<i32>> = Vec::with_capacity(polygon.len());
- for &(x, y) in polygon {
- if pts.last() != Some(&Point::new(x, y)) {
- pts.push(Point::new(x, y));
- }
- }
- while pts.len() > 1 && pts.first() == pts.last() {
- pts.pop();
- }
-
- match pts.as_slice() {
- [] => {}
- [p] => {
- if (0..width).contains(&p.x) && (0..height).contains(&p.y) {
- canvas.put_pixel(p.x as u32, p.y as u32, color);
- }
- }
- [a, b] => imageproc::drawing::draw_line_segment_mut(
- &mut canvas,
- (a.x as f32, a.y as f32),
- (b.x as f32, b.y as f32),
- color,
- ),
- _ => draw_polygon_mut(&mut canvas, &pts, color),
- }
- ImageU8::new(width, height, 1, canvas.into_raw())
-}
diff --git a/rust/crates/ml/src/cv/filter.rs b/rust/crates/ml/src/cv/filter.rs
deleted file mode 100644
index e4ae7f756e..0000000000
--- a/rust/crates/ml/src/cv/filter.rs
+++ /dev/null
@@ -1,114 +0,0 @@
-use crate::cv::OpResult;
-use crate::cv::image::{ImageF32, ImageU8};
-
-use super::reflect101;
-
-fn gaussian_taps(n: i32) -> OpResult<&'static [u32]> {
- Ok(match n {
- 1 => &[256],
- 3 => &[64, 128, 64],
- 5 => &[16, 64, 96, 64, 16],
- 7 => &[8, 28, 56, 72, 56, 28, 8],
- _ => return Err(format!("gaussian_blur_u8: unsupported ksize {n}")),
- })
-}
-
-pub(crate) fn gaussian_blur_u8(src: &ImageU8, ksize: i32) -> OpResult<ImageU8> {
- if src.channels != 1 {
- return Err("gaussian_blur_u8: expected a single-channel image".to_string());
- }
- let kw = if src.width == 1 { 1 } else { ksize };
- let kh = if src.height == 1 { 1 } else { ksize };
- if kw == 1 && kh == 1 {
- return Ok(src.clone());
- }
- let kx = gaussian_taps(kw)?;
- let ky = gaussian_taps(kh)?;
- let (w, h) = (src.width as usize, src.height as usize);
- let (ax, ay) = ((kw / 2) as i64, (kh / 2) as i64);
-
- let mut hsum = vec![0u16; w * h];
- for y in 0..h {
- let row = &src.data[y * w..(y + 1) * w];
- let out = &mut hsum[y * w..(y + 1) * w];
- for (x, o) in out.iter_mut().enumerate() {
- let mut acc = 0u16;
- for (t, &tap) in kx.iter().enumerate() {
- let sx = reflect101(x as i64 + t as i64 - ax, src.width) as usize;
- acc += tap as u16 * row[sx] as u16;
- }
- *o = acc;
- }
- }
- let mut out = vec![0u8; w * h];
- let mut acc = vec![0u32; w];
- for y in 0..h {
- let dst = &mut out[y * w..(y + 1) * w];
- acc.iter_mut().for_each(|a| *a = 0);
- for (t, &tap) in ky.iter().enumerate() {
- let sy = reflect101(y as i64 + t as i64 - ay, src.height) as usize;
- for (a, &v) in acc.iter_mut().zip(&hsum[sy * w..(sy + 1) * w]) {
- *a += tap * v as u32;
- }
- }
- for (d, &a) in dst.iter_mut().zip(acc.iter()) {
- *d = ((a + (1 << 15)) >> 16).min(255) as u8;
- }
- }
- ImageU8::new(src.width, src.height, 1, out)
-}
-
-pub(crate) fn box_filter_f32(src: &ImageF32, kw: i32, kh: i32) -> OpResult<ImageF32> {
- if kw <= 0 || kh <= 0 {
- return Err(format!("box_filter_f32: invalid kernel {kw}x{kh}"));
- }
- if src.channels != 1 {
- return Err("box_filter_f32: expected a single-channel image".to_string());
- }
- let (w, h) = (src.width as usize, src.height as usize);
- let (ax, ay) = ((kw / 2) as i64, (kh / 2) as i64);
- let scale = 1.0 / (kw as f64 * kh as f64);
-
- let mut hsum = vec![0.0f64; w * h];
- let ext_idx: Vec<usize> = (0..w + kw as usize - 1)
- .map(|j| reflect101(j as i64 - ax, src.width) as usize)
- .collect();
- let mut ext = vec![0.0f64; w + kw as usize - 1];
- for y in 0..h {
- let row = &src.data[y * w..(y + 1) * w];
- for (e, &sx) in ext.iter_mut().zip(ext_idx.iter()) {
- *e = row[sx] as f64;
- }
- let out = &mut hsum[y * w..(y + 1) * w];
- let mut s: f64 = ext[..kw as usize].iter().sum();
- out[0] = s;
- for x in 1..w {
- s += ext[x + kw as usize - 1] - ext[x - 1];
- out[x] = s;
- }
- }
-
- let mut out = vec![0.0f32; w * h];
- let mut col = vec![0.0f64; w];
- for t in 0..kh as usize - 1 {
- let sy = reflect101(t as i64 - ay, src.height) as usize;
- for (c, &v) in col.iter_mut().zip(&hsum[sy * w..(sy + 1) * w]) {
- *c += v;
- }
- }
- for y in 0..h {
- let enter = reflect101(y as i64 + kh as i64 - 1 - ay, src.height) as usize;
- for (c, &v) in col.iter_mut().zip(&hsum[enter * w..(enter + 1) * w]) {
- *c += v;
- }
- let dst = &mut out[y * w..(y + 1) * w];
- for (d, &c) in dst.iter_mut().zip(col.iter()) {
- *d = (c * scale) as f32;
- }
- let leave = reflect101(y as i64 - ay, src.height) as usize;
- for (c, &v) in col.iter_mut().zip(&hsum[leave * w..(leave + 1) * w]) {
- *c -= v;
- }
- }
- ImageF32::new(src.width, src.height, 1, out)
-}
diff --git a/rust/crates/ml/src/cv/image.rs b/rust/crates/ml/src/cv/image.rs
index 872b5ae603..8544abc583 100644
--- a/rust/crates/ml/src/cv/image.rs
+++ b/rust/crates/ml/src/cv/image.rs
@@ -1,6 +1,4 @@
-use rayon::prelude::*;
-
-use super::{OpResult, PARALLEL_MIN_ELEMS, saturate_u8_f32};
+use super::OpResult;
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct ImageU8 {
@@ -10,99 +8,15 @@ pub(crate) struct ImageU8 {
pub data: Vec<u8>,
}
-#[derive(Clone, Debug, PartialEq)]
-pub(crate) struct ImageF32 {
- pub width: i32,
- pub height: i32,
- pub channels: i32,
- pub data: Vec<f32>,
-}
-
-fn check_len(width: i32, height: i32, channels: i32, len: usize) -> OpResult<()> {
- if width <= 0 || height <= 0 || channels <= 0 {
- return Err(format!(
- "invalid image geometry {width}x{height}x{channels}"
- ));
- }
- let expected = width as usize * height as usize * channels as usize;
- if len != expected {
- return Err(format!(
- "image buffer length {len} does not match {width}x{height}x{channels} ({expected})"
- ));
- }
- Ok(())
-}
-
impl ImageU8 {
pub(crate) fn new(width: i32, height: i32, channels: i32, data: Vec<u8>) -> OpResult<Self> {
- check_len(width, height, channels, data.len())?;
- Ok(Self {
- width,
- height,
- channels,
- data,
- })
- }
-
- pub(crate) fn zeros(width: i32, height: i32, channels: i32) -> OpResult<Self> {
- if width <= 0 || height <= 0 || channels <= 0 {
+ let expected = checked_length(width, height, channels)?;
+ if data.len() != expected {
return Err(format!(
- "invalid image geometry {width}x{height}x{channels}"
+ "image buffer length {} does not match {width}x{height}x{channels} ({expected})",
+ data.len()
));
}
- let len = width as usize * height as usize * channels as usize;
- Ok(Self {
- width,
- height,
- channels,
- data: vec![0u8; len],
- })
- }
-
- pub(crate) fn size(&self) -> (i32, i32) {
- (self.width, self.height)
- }
-
- pub(crate) fn same_geometry(&self, other: &ImageU8) -> bool {
- self.width == other.width && self.height == other.height && self.channels == other.channels
- }
-
- pub(crate) fn to_f32(&self) -> ImageF32 {
- ImageF32 {
- width: self.width,
- height: self.height,
- channels: self.channels,
- data: map_vec(&self.data, |v| v as f32),
- }
- }
-}
-
-fn map_vec<T: Copy + Sync, U: Send>(src: &[T], f: impl Fn(T) -> U + Send + Sync) -> Vec<U> {
- if src.len() >= PARALLEL_MIN_ELEMS {
- src.par_iter().map(|&v| f(v)).collect()
- } else {
- src.iter().map(|&v| f(v)).collect()
- }
-}
-
-fn zip_vec<T: Copy + Sync, U: Send>(
- a: &[T],
- b: &[T],
- f: impl Fn(T, T) -> U + Send + Sync,
-) -> Vec<U> {
- if a.len() >= PARALLEL_MIN_ELEMS {
- a.par_iter()
- .zip(b.par_iter())
- .map(|(&x, &y)| f(x, y))
- .collect()
- } else {
- a.iter().zip(b.iter()).map(|(&x, &y)| f(x, y)).collect()
- }
-}
-
-impl ImageF32 {
- pub(crate) fn new(width: i32, height: i32, channels: i32, data: Vec<f32>) -> OpResult<Self> {
- check_len(width, height, channels, data.len())?;
Ok(Self {
width,
height,
@@ -111,97 +25,33 @@ impl ImageF32 {
})
}
+ #[cfg(test)]
pub(crate) fn zeros(width: i32, height: i32, channels: i32) -> OpResult<Self> {
- if width <= 0 || height <= 0 || channels <= 0 {
- return Err(format!(
- "invalid image geometry {width}x{height}x{channels}"
- ));
- }
- let len = width as usize * height as usize * channels as usize;
+ let length = checked_length(width, height, channels)?;
+ let mut data = Vec::new();
+ data.try_reserve_exact(length)
+ .map_err(|error| format!("image allocation failed: {error}"))?;
+ data.resize(length, 0);
Ok(Self {
width,
height,
channels,
- data: vec![0.0f32; len],
- })
- }
-
- pub(crate) fn pixels(&self) -> usize {
- self.width as usize * self.height as usize
- }
-
- pub(crate) fn same_geometry(&self, other: &ImageF32) -> bool {
- self.width == other.width && self.height == other.height && self.channels == other.channels
- }
-
- fn like(&self, data: Vec<f32>) -> ImageF32 {
- ImageF32 {
- width: self.width,
- height: self.height,
- channels: self.channels,
data,
- }
- }
-
- fn require_same(&self, other: &ImageF32) -> OpResult<()> {
- if self.same_geometry(other) {
- return Ok(());
- }
- Err(format!(
- "elementwise op on {}x{}x{} and {}x{}x{}",
- self.width, self.height, self.channels, other.width, other.height, other.channels
- ))
- }
-
- pub(crate) fn map(&self, f: impl Fn(f32) -> f32 + Send + Sync) -> ImageF32 {
- self.like(map_vec(&self.data, f))
- }
-
- pub(crate) fn map_mut(&mut self, f: impl Fn(f32) -> f32 + Send + Sync) {
- if self.data.len() >= PARALLEL_MIN_ELEMS {
- self.data.par_iter_mut().for_each(|v| *v = f(*v));
- } else {
- self.data.iter_mut().for_each(|v| *v = f(*v));
- }
- }
-
- pub(crate) fn map_to_u8(&self, f: impl Fn(f32) -> f32 + Send + Sync) -> ImageU8 {
- ImageU8 {
- width: self.width,
- height: self.height,
- channels: self.channels,
- data: map_vec(&self.data, |v| saturate_u8_f32(f(v))),
- }
- }
-
- pub(crate) fn zip_map(
- &self,
- other: &ImageF32,
- f: impl Fn(f32, f32) -> f32 + Send + Sync,
- ) -> OpResult<ImageF32> {
- self.require_same(other)?;
- Ok(self.like(zip_vec(&self.data, &other.data, f)))
+ })
}
+}
- pub(crate) fn zip_mut(
- &mut self,
- other: &ImageF32,
- f: impl Fn(&mut f32, f32) + Send + Sync,
- ) -> OpResult<()> {
- self.require_same(other)?;
- if self.data.len() >= PARALLEL_MIN_ELEMS {
- self.data
- .par_iter_mut()
- .zip(other.data.par_iter())
- .for_each(|(a, &b)| f(a, b));
- } else {
- self.data
- .iter_mut()
- .zip(other.data.iter())
- .for_each(|(a, &b)| f(a, b));
- }
- Ok(())
+fn checked_length(width: i32, height: i32, channels: i32) -> OpResult<usize> {
+ if width <= 0 || height <= 0 || channels <= 0 {
+ return Err(format!(
+ "invalid image geometry {width}x{height}x{channels}"
+ ));
}
+ (width as usize)
+ .checked_mul(height as usize)
+ .and_then(|n| n.checked_mul(channels as usize))
+ .filter(|&n| n <= isize::MAX as usize)
+ .ok_or_else(|| "image size overflow".to_owned())
}
#[derive(Clone, Debug, PartialEq)]
diff --git a/rust/crates/ml/src/cv/lab.rs b/rust/crates/ml/src/cv/lab.rs
deleted file mode 100644
index 6a4141bf92..0000000000
--- a/rust/crates/ml/src/cv/lab.rs
+++ /dev/null
@@ -1,243 +0,0 @@
-// L, a and b are all scaled onto 0..255 (a and b biased by 128); the
-// pipeline's chroma and luminance thresholds are calibrated in these units.
-
-use std::sync::OnceLock;
-
-use crate::cv::OpResult;
-use crate::cv::image::ImageU8;
-
-const LAB_SHIFT: i32 = 12;
-const LAB_SHIFT2: i32 = 15;
-const GAMMA_SHIFT: i32 = 3;
-const INV_GAMMA_SHIFT: i32 = 12;
-const INV_GAMMA_TAB_SIZE: usize = 1 << 12;
-const LAB_CBRT_TAB_SIZE: usize = 256 * 3 / 2 * (1 << GAMMA_SHIFT);
-const BASE_SHIFT: i32 = 14;
-const LAB_BASE: i32 = 1 << BASE_SHIFT;
-const MIN_AB_VALUE: i32 = -8145;
-const AB_TO_XZ_LEN: usize = (LAB_BASE as usize) * 9 / 4;
-const LAB2RGB_SHIFT: i32 = LAB_SHIFT + (BASE_SHIFT - INV_GAMMA_SHIFT);
-
-const SRGB2XYZ_D65: [f64; 9] = [
- 0.412453, 0.35758, 0.180423, 0.212671, 0.71516, 0.072169, 0.019334, 0.119193, 0.950227,
-];
-const XYZ2SRGB_D65: [f64; 9] = [
- 3.240479, -1.53715, -0.498535, -0.969256, 1.875991, 0.041556, 0.055648, -0.204043, 1.057311,
-];
-const D65: [f64; 3] = [0.950456, 1.0, 1.088754];
-
-const LTHRESH: f32 = 216.0 / 24389.0;
-const LSCALE: f32 = 841.0 / 108.0;
-const LBIAS: f32 = 16.0 / 116.0;
-
-const GAMMA_THRESHOLD: f64 = 809.0 / 20000.0;
-const GAMMA_INV_THRESHOLD: f64 = 7827.0 / 2500000.0;
-const GAMMA_LOW_SCALE: f64 = 323.0 / 25.0;
-const GAMMA_POWER: f64 = 12.0 / 5.0;
-const GAMMA_XSHIFT: f64 = 11.0 / 200.0;
-
-#[inline]
-fn descale(x: i32, n: i32) -> i32 {
- (x + (1 << (n - 1))) >> n
-}
-
-#[inline]
-fn round_i32(x: f64) -> i32 {
- x.round_ties_even() as i32
-}
-
-#[inline]
-fn saturate_u8(x: i32) -> u8 {
- x.clamp(0, 255) as u8
-}
-
-fn apply_gamma(x: f64) -> f64 {
- if x <= GAMMA_THRESHOLD {
- x / GAMMA_LOW_SCALE
- } else {
- ((x + GAMMA_XSHIFT) / (1.0 + GAMMA_XSHIFT)).powf(GAMMA_POWER)
- }
-}
-
-fn apply_inv_gamma(x: f64) -> f64 {
- if x <= GAMMA_INV_THRESHOLD {
- x * GAMMA_LOW_SCALE
- } else {
- x.powf(1.0 / GAMMA_POWER) * (1.0 + GAMMA_XSHIFT) - GAMMA_XSHIFT
- }
-}
-
-struct LabTabs {
- srgb_gamma: [u16; 256],
- lab_cbrt: [u16; LAB_CBRT_TAB_SIZE],
- srgb_inv_gamma: Box<[u16]>,
- lab_to_yf: [u16; 512],
- ab_to_xz: Box<[i32]>,
- fwd_coeffs: [i32; 9],
- inv_coeffs: [i32; 9],
-}
-
-fn build_tabs() -> LabTabs {
- let mut srgb_gamma = [0u16; 256];
- let int_scale = (255 * (1 << GAMMA_SHIFT)) as f64;
- for (i, out) in srgb_gamma.iter_mut().enumerate() {
- *out = round_i32(int_scale * apply_gamma(i as f64 / 255.0)) as u16;
- }
-
- let mut srgb_inv_gamma = vec![0u16; INV_GAMMA_TAB_SIZE];
- for (i, out) in srgb_inv_gamma.iter_mut().enumerate() {
- let x = i as f64 / INV_GAMMA_TAB_SIZE as f64;
- *out = round_i32(255.0 * apply_inv_gamma(x)) as u16;
- }
-
- let mut lab_cbrt = [0u16; LAB_CBRT_TAB_SIZE];
- let cb_tab_scale = 1.0f32 / (255.0 * (1 << GAMMA_SHIFT) as f32);
- let lshift2 = (1 << LAB_SHIFT2) as f32;
- for (i, out) in lab_cbrt.iter_mut().enumerate() {
- let x = cb_tab_scale * i as f32;
- let f = if x < LTHRESH {
- x.mul_add(LSCALE, LBIAS)
- } else {
- x.cbrt()
- };
- *out = round_i32((lshift2 * f) as f64) as u16;
- }
-
- let mut lab_to_yf = [0u16; 512];
- for i in 0..256i32 {
- const LUT_BASE: i32 = 1 << 14;
- let (y, ify) = if i <= 20 {
- (
- round_i32((i * LUT_BASE * 20 * 9) as f64 / (17 * 29 * 29 * 29) as f64),
- round_i32(LUT_BASE as f64 * (16.0 / 116.0 + (i * 5) as f64 / (3 * 17 * 29) as f64)),
- )
- } else {
- let fy =
- (i * 100 * LUT_BASE) as f64 / (255 * 116) as f64 + (16 * LUT_BASE) as f64 / 116.0;
- (
- round_i32(fy * fy * fy / (LUT_BASE * LUT_BASE) as f64),
- round_i32(fy),
- )
- };
- lab_to_yf[i as usize * 2] = y as u16;
- lab_to_yf[i as usize * 2 + 1] = ify as u16;
- }
-
- let mut ab_to_xz = vec![0i32; AB_TO_XZ_LEN];
- let bias = LAB_BASE * 16 / 116 * 108 / 841;
- for (k, out) in ab_to_xz.iter_mut().enumerate() {
- let i = k as i32 + MIN_AB_VALUE;
- *out = if i <= 3390 {
- i * 108 / 841 - bias
- } else {
- i * i / LAB_BASE * i / LAB_BASE
- };
- }
-
- let lshift = (1 << LAB_SHIFT) as f64;
- let mut fwd_coeffs = [0i32; 9];
- let mut inv_coeffs = [0i32; 9];
- for i in 0..3 {
- let wp = D65[i];
- fwd_coeffs[i * 3 + 2] = round_i32(lshift * SRGB2XYZ_D65[i * 3] / wp);
- fwd_coeffs[i * 3 + 1] = round_i32(lshift * SRGB2XYZ_D65[i * 3 + 1] / wp);
- fwd_coeffs[i * 3] = round_i32(lshift * SRGB2XYZ_D65[i * 3 + 2] / wp);
-
- inv_coeffs[i] = round_i32(lshift * XYZ2SRGB_D65[i] * wp);
- inv_coeffs[i + 3] = round_i32(lshift * XYZ2SRGB_D65[i + 3] * wp);
- inv_coeffs[i + 6] = round_i32(lshift * XYZ2SRGB_D65[i + 6] * wp);
- }
-
- LabTabs {
- srgb_gamma,
- lab_cbrt,
- srgb_inv_gamma: srgb_inv_gamma.into_boxed_slice(),
- lab_to_yf,
- ab_to_xz: ab_to_xz.into_boxed_slice(),
- fwd_coeffs,
- inv_coeffs,
- }
-}
-
-fn tabs() -> &'static LabTabs {
- static TABS: OnceLock<LabTabs> = OnceLock::new();
- TABS.get_or_init(build_tabs)
-}
-
-fn require_3ch(src: &ImageU8, op: &str) -> OpResult<()> {
- if src.channels != 3 {
- return Err(format!("{op}: expected 3 channel(s), got {}", src.channels));
- }
- Ok(())
-}
-
-pub(crate) fn bgr_to_lab(src: &ImageU8) -> OpResult<ImageU8> {
- require_3ch(src, "bgr_to_lab")?;
- let t = tabs();
- let c = t.fwd_coeffs;
- let l_scale = (116 * 255 + 50) / 100;
- let l_shift = -((16 * 255 * (1 << LAB_SHIFT2) + 50) / 100);
- let ab_shift = 128 * (1 << LAB_SHIFT2);
-
- let mut data = vec![0u8; src.data.len()];
- super::pointwise(&mut data, 3, &src.data, 3, |data, srcd| {
- for (out, px) in data
- .as_chunks_mut::<3>()
- .0
- .iter_mut()
- .zip(srcd.as_chunks::<3>().0.iter())
- {
- let v0 = t.srgb_gamma[px[0] as usize] as i32;
- let v1 = t.srgb_gamma[px[1] as usize] as i32;
- let v2 = t.srgb_gamma[px[2] as usize] as i32;
- let fx =
- t.lab_cbrt[descale(v0 * c[0] + v1 * c[1] + v2 * c[2], LAB_SHIFT) as usize] as i32;
- let fy =
- t.lab_cbrt[descale(v0 * c[3] + v1 * c[4] + v2 * c[5], LAB_SHIFT) as usize] as i32;
- let fz =
- t.lab_cbrt[descale(v0 * c[6] + v1 * c[7] + v2 * c[8], LAB_SHIFT) as usize] as i32;
-
- out[0] = saturate_u8(descale(l_scale * fy + l_shift, LAB_SHIFT2));
- out[1] = saturate_u8(descale(500 * (fx - fy) + ab_shift, LAB_SHIFT2));
- out[2] = saturate_u8(descale(200 * (fy - fz) + ab_shift, LAB_SHIFT2));
- }
- });
- ImageU8::new(src.width, src.height, 3, data)
-}
-
-pub(crate) fn lab_to_bgr(src: &ImageU8) -> OpResult<ImageU8> {
- require_3ch(src, "lab_to_bgr")?;
- let t = tabs();
- let c = t.inv_coeffs;
- let a_bias = 128 * LAB_BASE / 500;
- let b_bias = 128 * LAB_BASE / 200 - 1;
- let top = INV_GAMMA_TAB_SIZE as i32 - 1;
-
- let mut data = vec![0u8; src.data.len()];
- super::pointwise(&mut data, 3, &src.data, 3, |data, srcd| {
- for (out, px) in data
- .as_chunks_mut::<3>()
- .0
- .iter_mut()
- .zip(srcd.as_chunks::<3>().0.iter())
- {
- let y = t.lab_to_yf[px[0] as usize * 2] as i32;
- let ify = t.lab_to_yf[px[0] as usize * 2 + 1] as i32;
-
- let adiv = px[1] as i32 * LAB_BASE / 500 - a_bias;
- let bdiv = px[2] as i32 * LAB_BASE / 200 - b_bias;
-
- let x = t.ab_to_xz[(ify + adiv - MIN_AB_VALUE) as usize];
- let z = t.ab_to_xz[(ify - bdiv - MIN_AB_VALUE) as usize];
-
- let r = descale(c[0] * x + c[1] * y + c[2] * z, LAB2RGB_SHIFT).clamp(0, top);
- let g = descale(c[3] * x + c[4] * y + c[5] * z, LAB2RGB_SHIFT).clamp(0, top);
- let b = descale(c[6] * x + c[7] * y + c[8] * z, LAB2RGB_SHIFT).clamp(0, top);
-
- out[0] = saturate_u8(t.srgb_inv_gamma[b as usize] as i32);
- out[1] = saturate_u8(t.srgb_inv_gamma[g as usize] as i32);
- out[2] = saturate_u8(t.srgb_inv_gamma[r as usize] as i32);
- }
- });
- ImageU8::new(src.width, src.height, 3, data)
-}
diff --git a/rust/crates/ml/src/cv/masking.rs b/rust/crates/ml/src/cv/masking.rs
deleted file mode 100644
index 3d254efb76..0000000000
--- a/rust/crates/ml/src/cv/masking.rs
+++ /dev/null
@@ -1,78 +0,0 @@
-use crate::cv::OpResult;
-use crate::cv::image::ImageU8;
-
-pub(crate) fn threshold_binary_u8(src: &ImageU8, thresh: f64, maxval: f64) -> OpResult<ImageU8> {
- let ithresh = thresh.floor() as i64;
- let imaxval = maxval.round_ties_even().clamp(0.0, 255.0) as u8;
- ImageU8::new(
- src.width,
- src.height,
- src.channels,
- src.data
- .iter()
- .map(|&v| if v as i64 > ithresh { imaxval } else { 0 })
- .collect(),
- )
-}
-
-pub(crate) fn in_range_u8(src: &ImageU8, lower: f64, upper: f64) -> OpResult<ImageU8> {
- if src.channels != 1 {
- return Err("in_range_u8: expected a single-channel image".to_string());
- }
- let lo = lower.round_ties_even().clamp(0.0, 255.0) as u8;
- let hi = upper.round_ties_even().clamp(0.0, 255.0) as u8;
- ImageU8::new(
- src.width,
- src.height,
- 1,
- src.data
- .iter()
- .map(|&v| if lo <= v && v <= hi { 255 } else { 0 })
- .collect(),
- )
-}
-
-pub(crate) fn bitwise_and_u8(a: &ImageU8, b: &ImageU8) -> OpResult<ImageU8> {
- if !a.same_geometry(b) {
- return Err("bitwise_and_u8: operands have different geometry".to_string());
- }
- ImageU8::new(
- a.width,
- a.height,
- a.channels,
- a.data
- .iter()
- .zip(b.data.iter())
- .map(|(&x, &y)| x & y)
- .collect(),
- )
-}
-
-pub(crate) fn count_non_zero(src: &ImageU8) -> OpResult<i32> {
- if src.channels != 1 {
- return Err(format!(
- "count_non_zero: expected a single-channel image, got {}",
- src.channels
- ));
- }
- Ok(src.data.iter().filter(|&&v| v != 0).count() as i32)
-}
-
-pub(crate) fn copy_to_masked(src: &ImageU8, dst: &ImageU8, mask: &ImageU8) -> OpResult<ImageU8> {
- if !src.same_geometry(dst) {
- return Err("copy_to_masked: source and destination geometry differ".to_string());
- }
- if mask.channels != 1 || mask.width != src.width || mask.height != src.height {
- return Err("copy_to_masked: mask must be single-channel and the same size".to_string());
- }
- let cn = src.channels as usize;
- let mut out = dst.data.clone();
- super::pointwise3(&mut out, cn, &src.data, cn, &mask.data, 1, |o, s, m| {
- for (i, &m) in m.iter().enumerate() {
- if m != 0 {
- o[i * cn..(i + 1) * cn].copy_from_slice(&s[i * cn..(i + 1) * cn]);
- }
- }
- });
- ImageU8::new(src.width, src.height, src.channels, out)
-}
diff --git a/rust/crates/ml/src/cv/mod.rs b/rust/crates/ml/src/cv/mod.rs
index 556f2b0d18..82ce17f7a1 100644
--- a/rust/crates/ml/src/cv/mod.rs
+++ b/rust/crates/ml/src/cv/mod.rs
@@ -1,106 +1,14 @@
-mod bilateral;
-mod canny;
-mod channels;
mod contours;
-mod draw;
-mod filter;
pub(crate) mod image;
-mod lab;
-mod masking;
-mod morph;
+pub(crate) mod projective;
mod resize;
-mod stats;
-mod structuring;
mod transform;
mod warp;
mod warp_rgb;
-use rayon::prelude::*;
-
pub(crate) type OpResult<T> = Result<T, String>;
-pub(crate) use bilateral::bilateral_filter_u8;
-pub(crate) use canny::canny;
-pub(crate) use channels::{bgr_to_gray, gray_to_bgr, merge_u8, split_u8};
pub(crate) use contours::find_contours;
-pub(crate) use draw::fill_poly;
-pub(crate) use filter::{box_filter_f32, gaussian_blur_u8};
-pub(crate) use lab::{bgr_to_lab, lab_to_bgr};
-pub(crate) use masking::{
- bitwise_and_u8, copy_to_masked, count_non_zero, in_range_u8, threshold_binary_u8,
-};
-pub(crate) use morph::{morphology_close, morphology_erode, morphology_open};
-pub(crate) use resize::{Interp, resize_f32, resize_u8};
-pub(crate) use stats::{
- hist_256_f32, hist_256_u8, mean_f32, mean_u8c3_masked, min_max_loc_f32, percentile_f32,
- percentile_pair_f32,
-};
-pub(crate) use structuring::ellipse_kernel;
+pub(crate) use resize::{Interp, resize_u8};
pub(crate) use transform::rotate_u8;
-pub(crate) use warp::{warp_perspective, warp_rgb_perspective};
-
-pub(crate) const PARALLEL_MIN_ELEMS: usize = 200_000;
-const PIXELS_PER_CHUNK: usize = 65_536;
-
-pub(crate) fn pointwise<T: Sync, U: Send>(
- out: &mut [U],
- out_stride: usize,
- src: &[T],
- src_stride: usize,
- f: impl Fn(&mut [U], &[T]) + Send + Sync,
-) {
- if out.len() < PARALLEL_MIN_ELEMS {
- f(out, src);
- return;
- }
- out.par_chunks_mut(PIXELS_PER_CHUNK * out_stride)
- .zip(src.par_chunks(PIXELS_PER_CHUNK * src_stride))
- .for_each(|(o, s)| f(o, s));
-}
-
-pub(crate) fn pointwise3<T: Sync, V: Sync, U: Send>(
- out: &mut [U],
- out_stride: usize,
- a: &[T],
- a_stride: usize,
- b: &[V],
- b_stride: usize,
- f: impl Fn(&mut [U], &[T], &[V]) + Send + Sync,
-) {
- if out.len() < PARALLEL_MIN_ELEMS {
- f(out, a, b);
- return;
- }
- out.par_chunks_mut(PIXELS_PER_CHUNK * out_stride)
- .zip(a.par_chunks(PIXELS_PER_CHUNK * a_stride))
- .zip(b.par_chunks(PIXELS_PER_CHUNK * b_stride))
- .for_each(|((o, a), b)| f(o, a, b));
-}
-
-pub(crate) fn saturate_u8_f32(v: f32) -> u8 {
- v.round_ties_even().clamp(0.0, 255.0) as u8
-}
-
-pub(crate) fn saturate_u8_f64(v: f64) -> u8 {
- v.round_ties_even().clamp(0.0, 255.0) as u8
-}
-
-pub(crate) fn reflect101(mut p: i64, len: i32) -> i64 {
- let len = len as i64;
- if p >= 0 && p < len {
- return p;
- }
- if len == 1 {
- return 0;
- }
- loop {
- if p < 0 {
- p = -p;
- } else {
- p = len - 1 - (p - len) - 1;
- }
- if p >= 0 && p < len {
- return p;
- }
- }
-}
+pub(crate) use warp::warp_rgb_perspective;
diff --git a/rust/crates/ml/src/cv/morph.rs b/rust/crates/ml/src/cv/morph.rs
deleted file mode 100644
index c4809d34d9..0000000000
--- a/rust/crates/ml/src/cv/morph.rs
+++ /dev/null
@@ -1,75 +0,0 @@
-use rayon::prelude::*;
-
-use crate::cv::image::ImageU8;
-use crate::cv::{OpResult, PARALLEL_MIN_ELEMS};
-
-fn morph(src: &ImageU8, kernel: &ImageU8, erode: bool) -> OpResult<ImageU8> {
- if src.channels != 1 || kernel.channels != 1 {
- return Err("morphology: image and kernel must be single-channel".to_string());
- }
- if kernel.width * kernel.height == 1 {
- return Ok(src.clone());
- }
- let pad: u8 = if erode { 255 } else { 0 };
- let (w, h) = (src.width as usize, src.height as usize);
- let (kw, kh) = (kernel.width as usize, kernel.height as usize);
-
- let pw = w + kw - 1;
- let mut padded = vec![pad; pw * (h + kh - 1)];
- for y in 0..h {
- padded[(y + kh / 2) * pw + kw / 2..(y + kh / 2) * pw + kw / 2 + w]
- .copy_from_slice(&src.data[y * w..(y + 1) * w]);
- }
-
- let offsets: Vec<usize> = (0..kh * kw)
- .filter(|&i| kernel.data[i] != 0)
- .map(|i| (i / kw) * pw + i % kw)
- .collect();
- if offsets.is_empty() {
- return ImageU8::new(src.width, src.height, 1, vec![pad; w * h]);
- }
-
- let mut out = vec![0u8; w * h];
- let row = |y: usize, dst: &mut [u8]| {
- let base = y * pw;
- dst.copy_from_slice(&padded[base + offsets[0]..base + offsets[0] + w]);
- for &off in &offsets[1..] {
- let src_slice = &padded[base + off..base + off + w];
- if erode {
- for (d, &s) in dst.iter_mut().zip(src_slice) {
- *d = (*d).min(s);
- }
- } else {
- for (d, &s) in dst.iter_mut().zip(src_slice) {
- *d = (*d).max(s);
- }
- }
- }
- };
- if w * h >= PARALLEL_MIN_ELEMS {
- out.par_chunks_mut(w)
- .enumerate()
- .for_each(|(y, dst)| row(y, dst));
- } else {
- for (y, dst) in out.chunks_mut(w).enumerate() {
- row(y, dst);
- }
- }
- ImageU8::new(src.width, src.height, 1, out)
-}
-
-pub(crate) fn morphology_erode(src: &ImageU8, kernel: &ImageU8) -> OpResult<ImageU8> {
- morph(src, kernel, true)
-}
-
-fn morphology_dilate(src: &ImageU8, kernel: &ImageU8) -> OpResult<ImageU8> {
- morph(src, kernel, false)
-}
-
-pub(crate) fn morphology_open(src: &ImageU8, kernel: &ImageU8) -> OpResult<ImageU8> {
- morphology_dilate(&morphology_erode(src, kernel)?, kernel)
-}
-
-pub(crate) fn morphology_close(src: &ImageU8, kernel: &ImageU8) -> OpResult<ImageU8> {
- morphology_erode(&morphology_dilate(src, kernel)?, kernel)
-}
diff --git a/rust/crates/ml/src/cv/projective.rs b/rust/crates/ml/src/cv/projective.rs
new file mode 100644
index 0000000000..2dcd104025
--- /dev/null
+++ b/rust/crates/ml/src/cv/projective.rs
@@ -0,0 +1,287 @@
+use super::{OpResult, image::ImageU8};
+use rayon::prelude::*;
+
+#[derive(Debug, Clone, Copy)]
+pub(crate) struct ProjectiveMap {
+ pub coefficients: [f64; 9],
+}
+
+impl ProjectiveMap {
+ pub fn from_unit_quad(p: [(f64, f64); 4]) -> OpResult<Self> {
+ if p.iter().any(|(x, y)| !x.is_finite() || !y.is_finite()) {
+ return Err("non-finite projective coordinates".to_owned());
+ }
+ let origin = p[0];
+ let scale = p
+ .iter()
+ .map(|p| (p.0 - origin.0).hypot(p.1 - origin.1))
+ .fold(0.0, f64::max);
+ if scale <= f64::MIN_POSITIVE {
+ return Err("collapsed projective coordinates".to_owned());
+ }
+ let q = p.map(|p| ((p.0 - origin.0) / scale, (p.1 - origin.1) / scale));
+ let dx1 = q[1].0 - q[2].0;
+ let dx2 = q[3].0 - q[2].0;
+ let dy1 = q[1].1 - q[2].1;
+ let dy2 = q[3].1 - q[2].1;
+ let dx = q[0].0 - q[1].0 + q[2].0 - q[3].0;
+ let dy = q[0].1 - q[1].1 + q[2].1 - q[3].1;
+ let determinant = dx1 * dy2 - dx2 * dy1;
+ if determinant.abs() < 1e-12 {
+ return Err("ill-conditioned projective coordinates".to_owned());
+ }
+ let g = (dx * dy2 - dx2 * dy) / determinant;
+ let h = (dx1 * dy - dx * dy1) / determinant;
+ let denominators = [1.0, 1.0 + g, 1.0 + g + h, 1.0 + h];
+ let largest = denominators.iter().copied().fold(0.0, f64::max);
+ if denominators
+ .iter()
+ .any(|d| !d.is_finite() || *d < largest * 1e-6)
+ {
+ return Err("projective pole crosses or approaches the page".to_owned());
+ }
+ let coefficients = [
+ p[1].0 * (1.0 + g) - p[0].0,
+ p[3].0 * (1.0 + h) - p[0].0,
+ p[0].0,
+ p[1].1 * (1.0 + g) - p[0].1,
+ p[3].1 * (1.0 + h) - p[0].1,
+ p[0].1,
+ g,
+ h,
+ 1.0,
+ ];
+ Ok(Self { coefficients })
+ }
+
+ pub fn at(self, u: f64, v: f64) -> (f64, f64) {
+ let h = self.coefficients;
+ let d = h[6] * u + h[7] * v + 1.0;
+ (
+ (h[0] * u + h[1] * v + h[2]) / d,
+ (h[3] * u + h[4] * v + h[5]) / d,
+ )
+ }
+
+ fn footprint(self, u: f64, v: f64, width: i32, height: i32) -> (f64, f64) {
+ let h = self.coefficients;
+ let (x, y) = self.at(u, v);
+ let d = h[6] * u + h[7] * v + 1.0;
+ (
+ ((h[0] - h[6] * x).hypot(h[3] - h[6] * y) / d / width as f64).abs(),
+ ((h[1] - h[7] * x).hypot(h[4] - h[7] * y) / d / height as f64).abs(),
+ )
+ }
+}
+
+pub(crate) struct WarpedImage {
+ pub image: ImageU8,
+ pub valid: Option<Vec<u8>>,
+}
+
+fn buffer(length: usize, value: u8) -> OpResult<Vec<u8>> {
+ let mut data = Vec::new();
+ data.try_reserve_exact(length)
+ .map_err(|error| format!("image allocation failed: {error}"))?;
+ data.resize(length, value);
+ Ok(data)
+}
+
+fn reduce(source: &ImageU8) -> OpResult<ImageU8> {
+ let width = (source.width / 2).max(1);
+ let height = (source.height / 2).max(1);
+ super::resize_u8(source, width, height, super::Interp::Area)
+}
+
+fn sample(source: &ImageU8, x: f64, y: f64, original: &ImageU8) -> [f64; 3] {
+ let x = (x * source.width as f64 / original.width as f64 - 0.5)
+ .clamp(0.0, source.width as f64 - 1.0);
+ let y = (y * source.height as f64 / original.height as f64 - 0.5)
+ .clamp(0.0, source.height as f64 - 1.0);
+ let x0 = x.floor() as usize;
+ let y0 = y.floor() as usize;
+ let x1 = (x0 + 1).min(source.width as usize - 1);
+ let y1 = (y0 + 1).min(source.height as usize - 1);
+ let fx = x - x0 as f64;
+ let fy = y - y0 as f64;
+ std::array::from_fn(|c| {
+ let a = source.data[(y0 * source.width as usize + x0) * 3 + c] as f64;
+ let b = source.data[(y0 * source.width as usize + x1) * 3 + c] as f64;
+ let d = source.data[(y1 * source.width as usize + x0) * 3 + c] as f64;
+ let e = source.data[(y1 * source.width as usize + x1) * 3 + c] as f64;
+ (a + (b - a) * fx) * (1.0 - fy) + (d + (e - d) * fx) * fy
+ })
+}
+
+pub(crate) fn warp_document(
+ source: &ImageU8,
+ map: ProjectiveMap,
+ width: i32,
+ height: i32,
+) -> OpResult<WarpedImage> {
+ if source.width <= 0 || source.height <= 0 || source.channels != 3 || width <= 0 || height <= 0
+ {
+ return Err("invalid document warp geometry".to_owned());
+ }
+ let source_len = (source.width as usize)
+ .checked_mul(source.height as usize)
+ .and_then(|n| n.checked_mul(3));
+ if source_len != Some(source.data.len()) {
+ return Err("invalid document warp source buffer".to_owned());
+ }
+ let pixels = (width as usize)
+ .checked_mul(height as usize)
+ .ok_or("document size overflow")?;
+ let length = pixels.checked_mul(3).ok_or("document size overflow")?;
+ let mut data = buffer(length, 255)?;
+ let outside = |(x, y): (f64, f64)| {
+ x < 0.0 || y < 0.0 || x > source.width as f64 || y > source.height as f64
+ };
+ let has_missing = [(0.0, 0.0), (1.0, 0.0), (1.0, 1.0), (0.0, 1.0)]
+ .iter()
+ .any(|&(u, v)| outside(map.at(u, v)));
+ let mut validity = if has_missing {
+ Some(buffer(pixels, 255)?)
+ } else {
+ None
+ };
+ let mut max_footprint = 1.0f64;
+ for y in 0..=4 {
+ for x in 0..=4 {
+ let (a, b) = map.footprint(x as f64 / 4.0, y as f64 / 4.0, width, height);
+ max_footprint = max_footprint.max(a).max(b);
+ }
+ }
+ if !max_footprint.is_finite() {
+ return Err("non-finite projective footprint".to_owned());
+ }
+ let mut levels = Vec::new();
+ let mut factor = 1.0;
+ while factor * 4.0 < max_footprint {
+ let previous = levels.last().unwrap_or(source);
+ if previous.width == 1 && previous.height == 1 {
+ break;
+ }
+ levels.push(reduce(previous)?);
+ factor *= 2.0;
+ }
+ let render_row = |row: usize, output: &mut [u8], mut valid: Option<&mut [u8]>| {
+ for (column, pixel) in output.as_chunks_mut::<3>().0.iter_mut().enumerate() {
+ let u = (column as f64 + 0.5) / width as f64;
+ let v = (row as f64 + 0.5) / height as f64;
+ let (a, b) = map.footprint(u, v, width, height);
+ let level_index = ((a.max(b) / 2.0).log2().floor().max(0.0) as usize).min(levels.len());
+ let selected = if level_index == 0 {
+ source
+ } else {
+ &levels[level_index - 1]
+ };
+ let scale = (source.width as f64 / selected.width as f64)
+ .max(source.height as f64 / selected.height as f64);
+ let nx = (a / scale - 1e-9).ceil().clamp(1.0, 4.0) as usize;
+ let ny = (b / scale - 1e-9).ceil().clamp(1.0, 4.0) as usize;
+ let mut sum = [0.0; 3];
+ let mut complete = true;
+ for iy in 0..ny {
+ for ix in 0..nx {
+ let p = map.at(
+ (column as f64 + (ix as f64 + 0.5) / nx as f64) / width as f64,
+ (row as f64 + (iy as f64 + 0.5) / ny as f64) / height as f64,
+ );
+ let value = if outside(p) {
+ complete = false;
+ [255.0; 3]
+ } else {
+ sample(selected, p.0, p.1, source)
+ };
+ for c in 0..3 {
+ sum[c] += value[c];
+ }
+ }
+ }
+ for c in 0..3 {
+ pixel[c] = (sum[c] / (nx * ny) as f64).round().clamp(0.0, 255.0) as u8;
+ }
+ if let Some(valid) = valid.as_deref_mut() {
+ valid[column] = if complete { 255 } else { 0 };
+ }
+ }
+ };
+ let stride = width as usize * 3;
+ if pixels >= 262_144 {
+ if let Some(valid) = validity.as_mut() {
+ data.par_chunks_mut(stride)
+ .zip(valid.par_chunks_mut(width as usize))
+ .enumerate()
+ .for_each(|(row, (dst, valid))| render_row(row, dst, Some(valid)));
+ } else {
+ data.par_chunks_mut(stride)
+ .enumerate()
+ .for_each(|(row, dst)| render_row(row, dst, None));
+ }
+ } else {
+ for (row, dst) in data.chunks_mut(stride).enumerate() {
+ let valid = validity
+ .as_mut()
+ .map(|v| &mut v[row * width as usize..(row + 1) * width as usize]);
+ render_row(row, dst, valid);
+ }
+ }
+ Ok(WarpedImage {
+ image: ImageU8::new(width, height, 3, data)?,
+ valid: validity,
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn checkerboard_reduction_is_filtered() -> OpResult<()> {
+ let pixels = (0..64 * 64)
+ .flat_map(|i| [if (i / 64 + i % 64) % 2 == 0 { 0 } else { 255 }; 3])
+ .collect();
+ let source = ImageU8::new(64, 64, 3, pixels)?;
+ let map =
+ ProjectiveMap::from_unit_quad([(0.0, 0.0), (64.0, 0.0), (64.0, 64.0), (0.0, 64.0)])?;
+ let output = warp_document(&source, map, 7, 9)?;
+ assert!(output.image.data.iter().all(|v| (125..=130).contains(v)));
+ Ok(())
+ }
+
+ #[test]
+ fn unavailable_source_is_white_and_excluded_from_statistics() -> OpResult<()> {
+ let source = ImageU8::new(20, 20, 3, vec![50; 1200])?;
+ let map =
+ ProjectiveMap::from_unit_quad([(-2.0, 0.0), (20.0, 0.0), (20.0, 20.0), (-2.0, 20.0)])?;
+ let output = warp_document(&source, map, 22, 20)?;
+ assert_eq!(&output.image.data[0..6], &[255; 6]);
+ let valid = output.valid.ok_or("missing validity")?;
+ assert_eq!(&valid[..3], &[0, 0, 255]);
+ assert_eq!(&output.image.data[6..9], &[50; 3]);
+ Ok(())
+ }
+
+ #[test]
+ fn independently_projected_landmarks_match_mapping() -> OpResult<()> {
+ let project = |u: f64, v: f64| {
+ (
+ (120.0 * u + 24.0 * v + 17.0) / (0.3 * u + 0.2 * v + 1.0),
+ (13.0 * u + 180.0 * v + 29.0) / (0.3 * u + 0.2 * v + 1.0),
+ )
+ };
+ let map = ProjectiveMap::from_unit_quad([
+ project(0.0, 0.0),
+ project(1.0, 0.0),
+ project(1.0, 1.0),
+ project(0.0, 1.0),
+ ])?;
+ for (u, v) in [(0.0, 0.0), (0.17, 0.82), (0.5, 0.5), (1.0, 1.0)] {
+ let expected = project(u, v);
+ let actual = map.at(u, v);
+ assert!((actual.0 - expected.0).hypot(actual.1 - expected.1) < 1e-10);
+ }
+ Ok(())
+ }
+}
diff --git a/rust/crates/ml/src/cv/resize.rs b/rust/crates/ml/src/cv/resize.rs
index 1c8c49b192..872336a473 100644
--- a/rust/crates/ml/src/cv/resize.rs
+++ b/rust/crates/ml/src/cv/resize.rs
@@ -6,20 +6,18 @@ use fast_image_resize::{
};
use crate::cv::OpResult;
-use crate::cv::image::{ImageF32, ImageU8};
+use crate::cv::image::ImageU8;
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub(crate) enum Interp {
Bilinear,
Area,
- Bicubic,
}
impl Interp {
fn alg(self, upscaling: bool) -> ResizeAlg {
match self {
Interp::Bilinear => ResizeAlg::Interpolation(FilterType::Bilinear),
- Interp::Bicubic => ResizeAlg::Interpolation(FilterType::CatmullRom),
Interp::Area if upscaling => ResizeAlg::Interpolation(FilterType::Bilinear),
Interp::Area => ResizeAlg::Convolution(FilterType::Box),
}
@@ -80,30 +78,3 @@ pub(crate) fn resize_u8(
)?;
ImageU8::new(width, height, src.channels, data)
}
-
-pub(crate) fn resize_f32(
- src: &ImageF32,
- width: i32,
- height: i32,
- interp: Interp,
-) -> OpResult<ImageF32> {
- if width == src.width && height == src.height {
- return Ok(src.clone());
- }
- let bytes: Vec<u8> = src.data.iter().flat_map(|v| v.to_ne_bytes()).collect();
- let out = run(
- &bytes,
- (src.width, src.height),
- pixel_type(src.channels, true)?,
- width,
- height,
- interp.alg(width >= src.width && height >= src.height),
- )?;
- let data = out
- .as_chunks::<4>()
- .0
- .iter()
- .map(|b| f32::from_ne_bytes([b[0], b[1], b[2], b[3]]))
- .collect();
- ImageF32::new(width, height, src.channels, data)
-}
diff --git a/rust/crates/ml/src/cv/stats.rs b/rust/crates/ml/src/cv/stats.rs
deleted file mode 100644
index c3f045b61b..0000000000
--- a/rust/crates/ml/src/cv/stats.rs
+++ /dev/null
@@ -1,98 +0,0 @@
-use crate::cv::OpResult;
-use crate::cv::image::{ImageF32, ImageU8};
-
-fn sum_channel0(src: &ImageF32) -> f64 {
- let cn = src.channels as usize;
- if cn == 1 {
- src.data.iter().map(|&v| v as f64).sum()
- } else {
- src.data.chunks_exact(cn).map(|px| px[0] as f64).sum()
- }
-}
-
-pub(crate) fn mean_f32(src: &ImageF32) -> OpResult<f64> {
- Ok(sum_channel0(src) / src.pixels() as f64)
-}
-
-pub(crate) fn mean_u8c3_masked(src: &ImageU8, mask: &ImageU8) -> OpResult<[f64; 3]> {
- if src.channels != 3 {
- return Err(format!(
- "mean_u8c3_masked: expected a 3-channel image, got {}",
- src.channels
- ));
- }
- if mask.channels != 1 || mask.width != src.width || mask.height != src.height {
- return Err("mean_u8c3_masked: mask must be single-channel and the same size".to_string());
- }
- let mut sums = [0i64; 3];
- let mut count = 0i64;
- for (px, &m) in src.data.as_chunks::<3>().0.iter().zip(mask.data.iter()) {
- if m != 0 {
- sums[0] += px[0] as i64;
- sums[1] += px[1] as i64;
- sums[2] += px[2] as i64;
- count += 1;
- }
- }
- if count == 0 {
- return Ok([0.0; 3]);
- }
- Ok(sums.map(|s| s as f64 / count as f64))
-}
-
-pub(crate) fn min_max_loc_f32(src: &ImageF32) -> OpResult<(f64, f64)> {
- let mut min_val = f32::INFINITY;
- let mut max_val = f32::NEG_INFINITY;
- for &v in &src.data {
- if v < min_val {
- min_val = v;
- }
- if v > max_val {
- max_val = v;
- }
- }
- Ok((min_val as f64, max_val as f64))
-}
-
-pub(crate) fn hist_256_u8(src: &ImageU8) -> OpResult<Vec<f64>> {
- let cn = src.channels as usize;
- let mut bins = [0u32; 256];
- for px in src.data.chunks_exact(cn) {
- bins[px[0] as usize] += 1;
- }
- Ok(bins.iter().map(|&c| c as f64).collect())
-}
-
-pub(crate) fn hist_256_f32(src: &ImageF32) -> OpResult<Vec<f64>> {
- let cn = src.channels as usize;
- let mut bins = [0u32; 256];
- for px in src.data.chunks_exact(cn) {
- let v = px[0] as f64;
- if (0.0..256.0).contains(&v) {
- bins[v as usize] += 1;
- }
- }
- Ok(bins.iter().map(|&c| c as f64).collect())
-}
-
-fn nth(values: &mut [f32], p: f64) -> f32 {
- let index = ((values.len() as f64 * p) as usize).min(values.len() - 1);
- let (_, value, _) = values.select_nth_unstable_by(index, f32::total_cmp);
- *value
-}
-
-pub(crate) fn percentile_f32(src: &ImageF32, p: f64) -> OpResult<f32> {
- let mut values = src.data.clone();
- if values.is_empty() {
- return Err("percentile_f32: empty image".to_string());
- }
- Ok(nth(&mut values, p))
-}
-
-pub(crate) fn percentile_pair_f32(src: &ImageF32, low: f64, high: f64) -> OpResult<(f32, f32)> {
- let mut values = src.data.clone();
- if values.is_empty() {
- return Err("percentile_pair_f32: empty image".to_string());
- }
- Ok((nth(&mut values, low), nth(&mut values, high)))
-}
diff --git a/rust/crates/ml/src/cv/structuring.rs b/rust/crates/ml/src/cv/structuring.rs
deleted file mode 100644
index 12d8cd91cb..0000000000
--- a/rust/crates/ml/src/cv/structuring.rs
+++ /dev/null
@@ -1,37 +0,0 @@
-use crate::cv::OpResult;
-use crate::cv::image::ImageU8;
-
-pub(crate) fn ellipse_kernel(ksize: i32) -> OpResult<ImageU8> {
- if ksize <= 0 {
- return Err(format!("ellipse_kernel: invalid ksize {ksize}"));
- }
- let mut element = ImageU8::zeros(ksize, ksize, 1)?;
- let rect = ksize == 1;
- let r = ksize / 2;
- let inv_r2 = if r != 0 {
- 1.0 / (r as f64 * r as f64)
- } else {
- 0.0
- };
- let c = ksize / 2;
-
- for i in 0..ksize {
- let (mut j1, mut j2) = (0i32, 0i32);
- if rect {
- j2 = ksize;
- } else {
- let dy = i - r;
- if dy.abs() <= r {
- let dx = (c as f64 * (((r * r - dy * dy) as f64) * inv_r2).sqrt()).round_ties_even()
- as i32;
- j1 = (c - dx).max(0);
- j2 = (c + dx + 1).min(ksize);
- }
- }
- let row = (i * ksize) as usize;
- for j in j1.max(0)..j2 {
- element.data[row + j as usize] = 1;
- }
- }
- Ok(element)
-}
diff --git a/rust/crates/ml/src/cv/warp.rs b/rust/crates/ml/src/cv/warp.rs
index 4bdb7a6585..cd00984b88 100644
--- a/rust/crates/ml/src/cv/warp.rs
+++ b/rust/crates/ml/src/cv/warp.rs
@@ -4,29 +4,6 @@ use imageproc::geometric_transformations::Projection;
use crate::cv::OpResult;
use crate::cv::image::ImageU8;
-pub(crate) fn warp_perspective(
- src: &ImageU8,
- src_corners: [(f64, f64); 4],
- dst_corners: [(f64, f64); 4],
- width: i32,
- height: i32,
-) -> OpResult<ImageU8> {
- if width <= 0 || height <= 0 {
- return Err(format!(
- "warp_perspective: invalid destination size {width}x{height}"
- ));
- }
- if src.channels != 3 {
- return Err(format!(
- "warp_perspective: expected a 3-channel image, got {}",
- src.channels
- ));
- }
- let source = RgbImage::from_raw(src.width as u32, src.height as u32, src.data.clone())
- .ok_or_else(|| "warp_perspective: source buffer mismatch".to_string())?;
- warp_rgb_perspective(&source, src_corners, dst_corners, width, height)
-}
-
pub(crate) fn warp_rgb_perspective(
source: &RgbImage,
src_corners: [(f64, f64); 4],
diff --git a/rust/crates/ml/src/scan/appearance.rs b/rust/crates/ml/src/scan/appearance.rs
new file mode 100644
index 0000000000..e6c67d3c83
--- /dev/null
+++ b/rust/crates/ml/src/scan/appearance.rs
@@ -0,0 +1,1279 @@
+use std::sync::OnceLock;
+
+use crate::cv::OpResult;
+use crate::cv::image::ImageU8;
+
+use super::ColorMode;
+
+const ANALYSIS_LONG_EDGE: usize = 384;
+const CELL_EDGE: usize = 16;
+const CHROMA_BIN_COUNT: usize = 48;
+const PAPER_CHROMA_RADIUS: f32 = 0.032;
+const NEUTRAL_CHROMA_SPREAD: f32 = 0.034;
+const MIN_PAPER_LUMINANCE: f32 = 0.12;
+const MAX_CELL_LOG_STEP: f32 = 0.14;
+const MAX_PAPER_CURVATURE: f32 = 16.0;
+const ENCODE_TABLE_SIZE: usize = 8192;
+
+#[derive(Clone, Copy, Default)]
+struct Sample {
+ rgb: [f32; 3],
+ luminance: f32,
+ coverage: f32,
+ unclipped: f32,
+ smoothness: f32,
+}
+
+struct Analysis {
+ width: usize,
+ height: usize,
+ samples: Vec<Sample>,
+}
+
+#[derive(Clone, Copy, Default)]
+struct PaperCell {
+ log_luminance: f32,
+ weight: f32,
+}
+
+struct PaperField {
+ width: usize,
+ height: usize,
+ log_luminance: Vec<f32>,
+ confidence: Vec<f32>,
+ target_log_luminance: f32,
+ adaptation: [f32; 3],
+}
+
+struct TransferTables {
+ decode: [f32; 256],
+ encode: Vec<f32>,
+}
+
+fn transfer_tables() -> &'static TransferTables {
+ static TABLES: OnceLock<TransferTables> = OnceLock::new();
+ TABLES.get_or_init(|| TransferTables {
+ decode: std::array::from_fn(|value| {
+ let encoded = value as f32 / 255.0;
+ if encoded <= 0.04045 {
+ encoded / 12.92
+ } else {
+ ((encoded + 0.055) / 1.055).powf(2.4)
+ }
+ }),
+ encode: (0..=ENCODE_TABLE_SIZE)
+ .map(|index| {
+ let linear = index as f32 / ENCODE_TABLE_SIZE as f32;
+ 255.0
+ * if linear <= 0.0031308 {
+ linear * 12.92
+ } else {
+ 1.055 * linear.powf(1.0 / 2.4) - 0.055
+ }
+ })
+ .collect(),
+ })
+}
+
+fn luminance(rgb: [f32; 3]) -> f32 {
+ 0.2126 * rgb[0] + 0.7152 * rgb[1] + 0.0722 * rgb[2]
+}
+
+fn chromaticity(rgb: [f32; 3]) -> [f32; 3] {
+ let total = (rgb[0] + rgb[1] + rgb[2]).max(f32::MIN_POSITIVE);
+ rgb.map(|value| value / total)
+}
+
+fn chroma_distance(a: [f32; 3], b: [f32; 3]) -> f32 {
+ a.into_iter()
+ .zip(b)
+ .map(|(left, right)| (left - right).abs())
+ .fold(0.0, f32::max)
+}
+
+fn transition(value: f32, low: f32, high: f32) -> f32 {
+ let t = ((value - low) / (high - low)).clamp(0.0, 1.0);
+ t * t * (3.0 - 2.0 * t)
+}
+
+fn validate(image: &ImageU8, valid: Option<&[u8]>) -> OpResult<(usize, usize)> {
+ if image.width <= 0 || image.height <= 0 || image.channels != 3 {
+ return Err("document appearance requires a nonempty three-channel image".into());
+ }
+ let width = image.width as usize;
+ let height = image.height as usize;
+ let pixels = width
+ .checked_mul(height)
+ .ok_or_else(|| "document image dimensions overflow".to_string())?;
+ let bytes = pixels
+ .checked_mul(3)
+ .ok_or_else(|| "document image byte count overflows".to_string())?;
+ if image.data.len() != bytes {
+ return Err("document image buffer does not match its dimensions".into());
+ }
+ if valid.is_some_and(|mask| mask.len() != pixels) {
+ return Err("document validity mask does not match its image".into());
+ }
+ Ok((width, height))
+}
+
+fn analyze(image: &ImageU8, valid: Option<&[u8]>, long_edge: usize) -> Analysis {
+ let source_width = image.width as usize;
+ let source_height = image.height as usize;
+ let ratio = (long_edge as f64 / source_width.max(source_height) as f64).min(1.0);
+ let width = (source_width as f64 * ratio).round().max(1.0) as usize;
+ let height = (source_height as f64 * ratio).round().max(1.0) as usize;
+ let mut samples = vec![Sample::default(); width * height];
+ let decode = &transfer_tables().decode;
+ for y in 0..height {
+ let top = y * source_height / height;
+ let bottom = (y + 1) * source_height / height;
+ for x in 0..width {
+ let left = x * source_width / width;
+ let right = (x + 1) * source_width / width;
+ let mut rgb = [0.0; 3];
+ let mut count = 0;
+ let mut unclipped = 0;
+ for source_y in top..bottom {
+ for source_x in left..right {
+ let index = source_y * source_width + source_x;
+ if valid.is_some_and(|mask| mask[index] == 0) {
+ continue;
+ }
+ let pixel = &image.data[index * 3..index * 3 + 3];
+ unclipped += usize::from(pixel.iter().all(|&value| value < 254));
+ rgb[0] += decode[pixel[2] as usize];
+ rgb[1] += decode[pixel[1] as usize];
+ rgb[2] += decode[pixel[0] as usize];
+ count += 1;
+ }
+ }
+ if count != 0 {
+ rgb.iter_mut().for_each(|value| *value /= count as f32);
+ samples[y * width + x] = Sample {
+ rgb,
+ luminance: luminance(rgb),
+ coverage: count as f32 / ((right - left) * (bottom - top)) as f32,
+ unclipped: unclipped as f32 / count as f32,
+ smoothness: 0.0,
+ };
+ }
+ }
+ }
+ for y in 0..height {
+ for x in 0..width {
+ let index = y * width + x;
+ let center = samples[index];
+ if center.coverage < 0.6
+ || center.unclipped < 0.9
+ || center.luminance < MIN_PAPER_LUMINANCE
+ {
+ continue;
+ }
+ let mut low = center.luminance;
+ let mut high = center.luminance;
+ let mut chroma_range: f32 = 0.0;
+ let center_chroma = chromaticity(center.rgb);
+ for nearby_y in y.saturating_sub(1)..=(y + 1).min(height - 1) {
+ for nearby_x in x.saturating_sub(1)..=(x + 1).min(width - 1) {
+ let neighbor = samples[nearby_y * width + nearby_x];
+ if neighbor.coverage >= 0.6 {
+ low = low.min(neighbor.luminance);
+ high = high.max(neighbor.luminance);
+ chroma_range = chroma_range
+ .max(chroma_distance(center_chroma, chromaticity(neighbor.rgb)));
+ }
+ }
+ }
+ let local_range = (high - low) / center.luminance.max(MIN_PAPER_LUMINANCE);
+ samples[index].smoothness = (1.0 - transition(local_range, 0.07, 0.25))
+ * (1.0 - transition(chroma_range, 0.012, 0.055));
+ }
+ }
+ Analysis {
+ width,
+ height,
+ samples,
+ }
+}
+
+fn paper_chromaticity(analysis: &Analysis) -> Option<[f32; 3]> {
+ let mut histogram = vec![0.0f32; CHROMA_BIN_COUNT * CHROMA_BIN_COUNT];
+ for sample in &analysis.samples {
+ if sample.smoothness <= 0.0 {
+ continue;
+ }
+ let chroma = chromaticity(sample.rgb);
+ let x = (chroma[0] * CHROMA_BIN_COUNT as f32) as usize;
+ let y = (chroma[2] * CHROMA_BIN_COUNT as f32) as usize;
+ let x = x.min(CHROMA_BIN_COUNT - 1);
+ let y = y.min(CHROMA_BIN_COUNT - 1);
+ histogram[y * CHROMA_BIN_COUNT + x] +=
+ sample.smoothness * sample.coverage * sample.luminance.sqrt();
+ }
+ let mut peak = (0, 0);
+ let mut peak_weight = 0.0;
+ for y in 0..CHROMA_BIN_COUNT {
+ for x in 0..CHROMA_BIN_COUNT {
+ let mut weight = 0.0;
+ for near_y in y.saturating_sub(1)..=(y + 1).min(CHROMA_BIN_COUNT - 1) {
+ for near_x in x.saturating_sub(1)..=(x + 1).min(CHROMA_BIN_COUNT - 1) {
+ let horizontal = if near_x == x { 2.0 } else { 1.0 };
+ let vertical = if near_y == y { 2.0 } else { 1.0 };
+ weight += histogram[near_y * CHROMA_BIN_COUNT + near_x] * horizontal * vertical;
+ }
+ }
+ if weight > peak_weight {
+ peak = (x, y);
+ peak_weight = weight;
+ }
+ }
+ }
+ if peak_weight == 0.0 {
+ return None;
+ }
+ let red = (peak.0 as f32 + 0.5) / CHROMA_BIN_COUNT as f32;
+ let blue = (peak.1 as f32 + 0.5) / CHROMA_BIN_COUNT as f32;
+ let mut center = [red, 1.0 - red - blue, blue];
+ for _ in 0..3 {
+ let mut sum = [0.0; 3];
+ let mut weight = 0.0;
+ for sample in &analysis.samples {
+ let chroma = chromaticity(sample.rgb);
+ if sample.smoothness > 0.0 && chroma_distance(center, chroma) < PAPER_CHROMA_RADIUS {
+ let w = sample.smoothness * sample.coverage;
+ for channel in 0..3 {
+ sum[channel] += chroma[channel] * w;
+ }
+ weight += w;
+ }
+ }
+ if weight == 0.0 {
+ return None;
+ }
+ center = chromaticity(sum);
+ }
+ Some(center)
+}
+
+fn estimate_field(analysis: &Analysis) -> Option<PaperField> {
+ let chroma = paper_chromaticity(analysis)?;
+ let width = analysis.width.div_ceil(CELL_EDGE);
+ let height = analysis.height.div_ceil(CELL_EDGE);
+ let mut cells = vec![PaperCell::default(); width * height];
+ let mut eligible = Vec::with_capacity(CELL_EDGE * CELL_EDGE);
+ let mut valid_weight = 0.0;
+ let mut paper_weight = 0.0;
+ for y in 0..height {
+ for x in 0..width {
+ eligible.clear();
+ let left = x * analysis.width / width;
+ let right = (x + 1) * analysis.width / width;
+ let top = y * analysis.height / height;
+ let bottom = (y + 1) * analysis.height / height;
+ let mut supported = 0.0;
+ let mut available = 0.0;
+ for sample_y in top..bottom {
+ for sample_x in left..right {
+ let sample = analysis.samples[sample_y * analysis.width + sample_x];
+ available += sample.coverage;
+ if sample.smoothness > 0.5
+ && chroma_distance(chromaticity(sample.rgb), chroma) < PAPER_CHROMA_RADIUS
+ {
+ eligible.push(sample.luminance);
+ supported += sample.coverage;
+ }
+ }
+ }
+ valid_weight += available;
+ paper_weight += supported;
+ if supported >= available * 0.18 && eligible.len() >= 3 {
+ eligible.sort_unstable_by(f32::total_cmp);
+ let bright = eligible[eligible.len() * 3 / 4];
+ let median = eligible[eligible.len() / 2];
+ let spread = (bright / median).ln();
+ cells[y * width + x] = PaperCell {
+ log_luminance: bright.ln(),
+ weight: (supported / available.max(1.0))
+ * (1.0 - transition(spread, 0.06, 0.22)),
+ };
+ }
+ }
+ }
+ if valid_weight < 12.0 || paper_weight / valid_weight < 0.38 {
+ return None;
+ }
+ retain_paper_component(&mut cells, width, height);
+ if paper_curvature(&cells, width, height) > MAX_PAPER_CURVATURE {
+ return None;
+ }
+ let supported_cells = cells.iter().filter(|cell| cell.weight > 0.0).count();
+ if supported_cells < 2 || supported_cells as f32 / cells.len() as f32 <= 0.32 {
+ return None;
+ }
+ let mut regions = [false; 9];
+ for y in 0..height {
+ for x in 0..width {
+ if cells[y * width + x].weight > 0.0 {
+ regions[(y * 3 / height).min(2) * 3 + (x * 3 / width).min(2)] = true;
+ }
+ }
+ }
+ if width >= 3 && height >= 3 && regions.iter().filter(|&&value| value).count() < 7 {
+ return None;
+ }
+ let mut measured: Vec<f32> = cells
+ .iter()
+ .filter(|cell| cell.weight > 0.0)
+ .map(|cell| cell.log_luminance)
+ .collect();
+ measured.sort_unstable_by(f32::total_cmp);
+ let initial = measured[measured.len() / 2];
+ let bright = measured[measured.len() * 9 / 10];
+ let neutral = chroma.iter().copied().fold(f32::NEG_INFINITY, f32::max)
+ - chroma.iter().copied().fold(f32::INFINITY, f32::min)
+ < NEUTRAL_CHROMA_SPREAD;
+ let target_log_luminance = if neutral {
+ bright.max(0.965f32.ln())
+ } else {
+ bright
+ };
+ let mut field = vec![initial; width * height];
+ let mut scratch = field.clone();
+ let mut distance = vec![width + height; width * height];
+ for (index, cell) in cells.iter().enumerate() {
+ if cell.weight > 0.0 {
+ field[index] = cell.log_luminance;
+ distance[index] = 0;
+ }
+ }
+ for _ in 0..(width + height) {
+ let mut changed = false;
+ for y in 0..height {
+ for x in 0..width {
+ let index = y * width + x;
+ let old = distance[index];
+ for neighbor in neighbors(x, y, width, height).into_iter().flatten() {
+ distance[index] = distance[index].min(distance[neighbor] + 1);
+ }
+ changed |= old != distance[index];
+ }
+ }
+ if !changed {
+ break;
+ }
+ }
+ for _ in 0..(width.max(height) * 6) {
+ for y in 0..height {
+ for x in 0..width {
+ let index = y * width + x;
+ let evidence = cells[index].weight * 7.0;
+ let mut sum = cells[index].log_luminance * evidence;
+ let mut total = evidence;
+ for neighbor in neighbors(x, y, width, height).into_iter().flatten() {
+ sum += field[neighbor];
+ total += 1.0;
+ }
+ scratch[index] = if total > 0.0 { sum / total } else { initial };
+ }
+ }
+ std::mem::swap(&mut field, &mut scratch);
+ }
+ let confidence_start = (width.min(height) as f32 * 0.3).max(2.0);
+ let confidence_end = (width.max(height) as f32 * 0.75).max(confidence_start + 1.0);
+ let confidence = distance
+ .into_iter()
+ .map(|value| 1.0 - transition(value as f32, confidence_start, confidence_end))
+ .collect();
+ let adaptation = if neutral {
+ let paper_luminance = luminance(chroma);
+ chroma.map(|channel| (paper_luminance / channel.max(0.001)).clamp(0.88, 1.12))
+ } else {
+ [1.0; 3]
+ };
+ Some(PaperField {
+ width,
+ height,
+ log_luminance: field,
+ confidence,
+ target_log_luminance,
+ adaptation,
+ })
+}
+
+fn neighbors(x: usize, y: usize, width: usize, height: usize) -> [Option<usize>; 4] {
+ [
+ (x > 0).then(|| y * width + x - 1),
+ (x + 1 < width).then_some(y * width + x + 1),
+ (y > 0).then(|| (y - 1) * width + x),
+ (y + 1 < height).then_some((y + 1) * width + x),
+ ]
+}
+
+fn paper_curvature(cells: &[PaperCell], width: usize, height: usize) -> f32 {
+ let mut curvature = Vec::new();
+ for y in 0..height {
+ for x in 0..width {
+ let center = cells[y * width + x];
+ if center.weight == 0.0 {
+ continue;
+ }
+ if x > 0 && x + 1 < width {
+ let before = cells[y * width + x - 1];
+ let after = cells[y * width + x + 1];
+ if before.weight > 0.0 && after.weight > 0.0 {
+ curvature.push(
+ (before.log_luminance + after.log_luminance - 2.0 * center.log_luminance)
+ .abs()
+ * (width * width) as f32,
+ );
+ }
+ }
+ if y > 0 && y + 1 < height {
+ let before = cells[(y - 1) * width + x];
+ let after = cells[(y + 1) * width + x];
+ if before.weight > 0.0 && after.weight > 0.0 {
+ curvature.push(
+ (before.log_luminance + after.log_luminance - 2.0 * center.log_luminance)
+ .abs()
+ * (height * height) as f32,
+ );
+ }
+ }
+ }
+ }
+ curvature.sort_unstable_by(f32::total_cmp);
+ if curvature.is_empty() {
+ 0.0
+ } else {
+ curvature[curvature.len() * 3 / 4]
+ }
+}
+
+fn retain_paper_component(cells: &mut [PaperCell], width: usize, height: usize) {
+ let mut labels = vec![usize::MAX; cells.len()];
+ let mut queue = Vec::with_capacity(cells.len());
+ let mut best_label = 0;
+ let mut best_weight = 0.0;
+ let mut label = 0;
+ for start in 0..cells.len() {
+ if cells[start].weight <= 0.0 || labels[start] != usize::MAX {
+ continue;
+ }
+ queue.clear();
+ queue.push(start);
+ labels[start] = label;
+ let mut index = 0;
+ let mut weight = 0.0;
+ while index < queue.len() {
+ let current = queue[index];
+ weight += cells[current].weight * (0.3 * cells[current].log_luminance).exp();
+ for neighbor in neighbors(current % width, current / width, width, height)
+ .into_iter()
+ .flatten()
+ {
+ if cells[neighbor].weight > 0.0
+ && labels[neighbor] == usize::MAX
+ && paper_cells_agree(cells, current, neighbor, width, height)
+ {
+ labels[neighbor] = label;
+ queue.push(neighbor);
+ }
+ }
+ index += 1;
+ }
+ if weight > best_weight {
+ best_weight = weight;
+ best_label = label;
+ }
+ label += 1;
+ }
+ for (cell, label) in cells.iter_mut().zip(labels) {
+ if label != best_label {
+ cell.weight = 0.0;
+ }
+ }
+}
+
+fn paper_cells_agree(
+ cells: &[PaperCell],
+ current: usize,
+ neighbor: usize,
+ width: usize,
+ height: usize,
+) -> bool {
+ let step = (cells[neighbor].log_luminance - cells[current].log_luminance).abs();
+ if step > MAX_CELL_LOG_STEP {
+ return false;
+ }
+ let current_x = (current % width) as isize;
+ let current_y = (current / width) as isize;
+ let neighbor_x = (neighbor % width) as isize;
+ let neighbor_y = (neighbor / width) as isize;
+ let dx = neighbor_x - current_x;
+ let dy = neighbor_y - current_y;
+ let before = (current_x - dx, current_y - dy);
+ let after = (neighbor_x + dx, neighbor_y + dy);
+ let at = |(x, y): (isize, isize)| {
+ (x >= 0 && y >= 0 && x < width as isize && y < height as isize)
+ .then(|| cells[y as usize * width + x as usize])
+ .filter(|cell| cell.weight > 0.0)
+ };
+ if let (Some(before), Some(after)) = (at(before), at(after)) {
+ let slope = (cells[current].log_luminance - before.log_luminance)
+ .abs()
+ .max((after.log_luminance - cells[neighbor].log_luminance).abs());
+ step <= slope * 2.0 + 0.035
+ } else {
+ true
+ }
+}
+
+fn interpolate(values: &[f32], width: usize, height: usize, x: f32, y: f32) -> f32 {
+ let x = (x * width as f32 - 0.5).clamp(0.0, (width - 1) as f32);
+ let y = (y * height as f32 - 0.5).clamp(0.0, (height - 1) as f32);
+ let left = x as usize;
+ let top = y as usize;
+ let right = (left + 1).min(width - 1);
+ let bottom = (top + 1).min(height - 1);
+ let fx = x - left as f32;
+ let fy = y - top as f32;
+ let upper = values[top * width + left] * (1.0 - fx) + values[top * width + right] * fx;
+ let lower = values[bottom * width + left] * (1.0 - fx) + values[bottom * width + right] * fx;
+ upper * (1.0 - fy) + lower * fy
+}
+
+fn encode(value: f32, table: &[f32]) -> u8 {
+ let index = value.clamp(0.0, 1.0) * ENCODE_TABLE_SIZE as f32;
+ let low = index as usize;
+ let high = (low + 1).min(ENCODE_TABLE_SIZE);
+ let fraction = index - low as f32;
+ (table[low] * (1.0 - fraction) + table[high] * fraction).round() as u8
+}
+
+fn render(image: &mut ImageU8, valid: Option<&[u8]>, field: Option<&PaperField>, grayscale: bool) {
+ let width = image.width as usize;
+ let height = image.height as usize;
+ let tables = transfer_tables();
+ for (index, pixel) in image.data.as_chunks_mut::<3>().0.iter_mut().enumerate() {
+ if valid.is_some_and(|mask| mask[index] == 0) {
+ pixel.fill(255);
+ continue;
+ }
+ if field.is_none() && !grayscale {
+ continue;
+ }
+ let mut rgb = [
+ tables.decode[pixel[2] as usize],
+ tables.decode[pixel[1] as usize],
+ tables.decode[pixel[0] as usize],
+ ];
+ if let Some(field) = field {
+ let x = (index % width) as f32 / width as f32 + 0.5 / width as f32;
+ let y = (index / width) as f32 / height as f32 + 0.5 / height as f32;
+ let paper = interpolate(&field.log_luminance, field.width, field.height, x, y);
+ let confidence = interpolate(&field.confidence, field.width, field.height, x, y);
+ let gain =
+ ((field.target_log_luminance - paper).clamp(-0.08, 4.0f32.ln()) * confidence).exp();
+ for (value, adaptation) in rgb.iter_mut().zip(field.adaptation) {
+ *value *= gain * (1.0 + (adaptation - 1.0) * confidence);
+ }
+ let maximum = rgb.iter().copied().fold(1.0, f32::max);
+ rgb.iter_mut().for_each(|value| *value /= maximum);
+ }
+ if grayscale {
+ pixel.fill(encode(luminance(rgb), &tables.encode));
+ } else {
+ pixel[0] = encode(rgb[2], &tables.encode);
+ pixel[1] = encode(rgb[1], &tables.encode);
+ pixel[2] = encode(rgb[0], &tables.encode);
+ }
+ }
+}
+
+fn meaningful_color(image: &ImageU8, valid: Option<&[u8]>) -> bool {
+ let width = image.width as usize;
+ let height = image.height as usize;
+ let mut coherent = 0;
+ for y in 0..height {
+ for x in 0..width {
+ let index = y * width + x;
+ if valid.is_some_and(|mask| mask[index] == 0) {
+ continue;
+ }
+ let pixel = &image.data[index * 3..index * 3 + 3];
+ let low = pixel[0].min(pixel[1]).min(pixel[2]) as i16;
+ let high = pixel[0].max(pixel[1]).max(pixel[2]) as i16;
+ if high - low < 13 {
+ continue;
+ }
+ if width * height < 36 {
+ return true;
+ }
+ let differences = [
+ pixel[2] as i16 - pixel[1] as i16,
+ pixel[0] as i16 - pixel[1] as i16,
+ ];
+ let mut matched = false;
+ for neighbor in [
+ (x + 1 < width).then_some(index + 1),
+ (y + 1 < height).then_some(index + width),
+ ]
+ .into_iter()
+ .flatten()
+ {
+ if valid.is_some_and(|mask| mask[neighbor] == 0) {
+ continue;
+ }
+ let other = &image.data[neighbor * 3..neighbor * 3 + 3];
+ let other_differences = [
+ other[2] as i16 - other[1] as i16,
+ other[0] as i16 - other[1] as i16,
+ ];
+ let dot = differences[0] as i32 * other_differences[0] as i32
+ + differences[1] as i32 * other_differences[1] as i32;
+ let length = differences[0] as i32 * differences[0] as i32
+ + differences[1] as i32 * differences[1] as i32;
+ matched |= dot > length / 2;
+ }
+ if matched {
+ coherent += 1;
+ if coherent >= 6 {
+ return true;
+ }
+ }
+ }
+ }
+ false
+}
+
+pub(super) fn render_document(
+ image: &mut ImageU8,
+ valid: Option<&[u8]>,
+ requested_mode: Option<ColorMode>,
+) -> OpResult<ColorMode> {
+ validate(image, valid)?;
+ let analysis = analyze(image, valid, ANALYSIS_LONG_EDGE);
+ let field = estimate_field(&analysis);
+ let explicit_grayscale = matches!(requested_mode, Some(ColorMode::Grayscale));
+ render(image, valid, field.as_ref(), explicit_grayscale);
+ let mode = requested_mode.unwrap_or_else(|| {
+ if meaningful_color(image, valid) {
+ ColorMode::Color
+ } else {
+ ColorMode::Grayscale
+ }
+ });
+ if requested_mode.is_none() && matches!(mode, ColorMode::Grayscale) {
+ render(image, valid, None, true);
+ }
+ Ok(mode)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[derive(Clone, Copy, PartialEq, Eq)]
+ enum Region {
+ Paper,
+ Ink,
+ Faint,
+ Color,
+ Illustration,
+ }
+
+ struct Fixture {
+ clean: ImageU8,
+ shaded: ImageU8,
+ regions: Vec<Region>,
+ }
+
+ fn process(image: &mut ImageU8, valid: Option<&[u8]>, mode: Option<ColorMode>) -> ColorMode {
+ match render_document(image, valid, mode) {
+ Ok(mode) => mode,
+ Err(error) => panic!("{error}"),
+ }
+ }
+
+ fn image(width: usize, height: usize, rgb: [u8; 3]) -> ImageU8 {
+ ImageU8 {
+ width: width as i32,
+ height: height as i32,
+ channels: 3,
+ data: [rgb[2], rgb[1], rgb[0]].repeat(width * height),
+ }
+ }
+
+ fn document(width: usize, height: usize, stock: [u8; 3], color: bool) -> Fixture {
+ let mut clean = image(width, height, stock);
+ let mut shaded = clean.clone();
+ let mut regions = vec![Region::Paper; width * height];
+ let transfer = transfer_tables();
+ for y in 0..height {
+ for x in 0..width {
+ let index = y * width + x;
+ let mut rgb = stock;
+ let nx = x as f32 / width as f32;
+ let ny = y as f32 / height as f32;
+ if (0.09..0.84).contains(&nx)
+ && (0.10..0.66).contains(&ny)
+ && y % 29 < 3
+ && x % 17 < 11
+ {
+ rgb = [38; 3];
+ regions[index] = Region::Ink;
+ }
+ if (0.11..0.83).contains(&nx) && y == height * 7 / 10 {
+ rgb = stock.map(|value| value.saturating_sub(15));
+ regions[index] = Region::Faint;
+ }
+ if color && (0.77..0.87).contains(&ny) && (0.08..0.89).contains(&nx) {
+ let patch = ((nx - 0.08) / 0.21) as usize;
+ rgb = [[185, 33, 58], [21, 103, 206], [30, 163, 88], [239, 180, 38]]
+ [patch.min(3)];
+ regions[index] = Region::Color;
+ }
+ if color
+ && (0.34..0.37).contains(&ny)
+ && (0.08..0.89).contains(&nx)
+ && regions[index] == Region::Paper
+ {
+ rgb = [244, 235, 120];
+ regions[index] = Region::Color;
+ }
+ let illumination = 0.37 + 0.55 * nx + 0.045 * (ny * 5.0).sin();
+ for (channel, value) in rgb.into_iter().enumerate() {
+ clean.data[index * 3 + 2 - channel] = value;
+ shaded.data[index * 3 + 2 - channel] = encode(
+ transfer.decode[value as usize] * illumination,
+ &transfer.encode,
+ );
+ }
+ }
+ }
+ Fixture {
+ clean,
+ shaded,
+ regions,
+ }
+ }
+
+ fn region_mean(image: &ImageU8, regions: &[Region], region: Region) -> f32 {
+ let mut sum = 0.0;
+ let mut count = 0;
+ for (pixel, &kind) in image.data.as_chunks::<3>().0.iter().zip(regions) {
+ if kind == region {
+ sum += pixel.iter().map(|&value| value as f32).sum::<f32>() / 3.0;
+ count += 1;
+ }
+ }
+ sum / count as f32
+ }
+
+ fn region_error(
+ image: &ImageU8,
+ reference: &ImageU8,
+ regions: &[Region],
+ region: Region,
+ ) -> f32 {
+ let mut sum = 0.0;
+ let mut count = 0;
+ for ((pixel, expected), &kind) in image
+ .data
+ .as_chunks::<3>()
+ .0
+ .iter()
+ .zip(reference.data.as_chunks::<3>().0.iter())
+ .zip(regions)
+ {
+ if kind == region {
+ for channel in 0..3 {
+ sum += (pixel[channel] as f32 - expected[channel] as f32).abs();
+ count += 1;
+ }
+ }
+ }
+ sum / count as f32
+ }
+
+ fn paper_deviation(image: &ImageU8, regions: &[Region]) -> f32 {
+ let mean = region_mean(image, regions, Region::Paper);
+ let mut squared = 0.0;
+ let mut count = 0;
+ for (pixel, &kind) in image.data.as_chunks::<3>().0.iter().zip(regions) {
+ if kind == Region::Paper {
+ let value = pixel.iter().map(|&value| value as f32).sum::<f32>() / 3.0;
+ squared += (value - mean).powi(2);
+ count += 1;
+ }
+ }
+ (squared / count as f32).sqrt()
+ }
+
+ #[test]
+ fn restores_shaded_color_paper_without_erasing_faint_strokes() {
+ let fixture = document(512, 704, [248; 3], true);
+ let mut rendered = fixture.shaded.clone();
+ let mode = process(&mut rendered, None, None);
+ let paper_error = region_error(&rendered, &fixture.clean, &fixture.regions, Region::Paper);
+ let color_error = region_error(&rendered, &fixture.clean, &fixture.regions, Region::Color);
+ let deviation = paper_deviation(&rendered, &fixture.regions);
+ let faint_contrast = region_mean(&rendered, &fixture.regions, Region::Paper)
+ - region_mean(&rendered, &fixture.regions, Region::Faint);
+ eprintln!(
+ "neutral page: paper MAE {paper_error:.3}, color MAE {color_error:.3}, paper SD {deviation:.3}, faint contrast {faint_contrast:.3}"
+ );
+ assert!(matches!(mode, ColorMode::Color));
+ assert!(paper_error < 7.0);
+ assert!(color_error < 8.0);
+ assert!(deviation < paper_deviation(&fixture.shaded, &fixture.regions) * 0.2);
+ assert!(faint_contrast > 11.0);
+ assert!(region_mean(&rendered, &fixture.regions, Region::Ink) < 48.0);
+ }
+
+ #[test]
+ fn color_and_grayscale_share_the_same_paper_correction() {
+ let fixture = document(384, 512, [248; 3], false);
+ let mut color = fixture.shaded.clone();
+ let mut gray = fixture.shaded;
+ assert!(matches!(
+ process(&mut color, None, Some(ColorMode::Color)),
+ ColorMode::Color
+ ));
+ assert!(matches!(
+ process(&mut gray, None, Some(ColorMode::Grayscale)),
+ ColorMode::Grayscale
+ ));
+ assert_eq!(color.data, gray.data);
+ assert!(paper_deviation(&gray, &fixture.regions) < 4.0);
+ }
+
+ #[test]
+ fn shaded_colored_stock_preserves_its_tint_and_flattens_light() {
+ let fixture = document(416, 608, [235, 211, 159], true);
+ let mut rendered = fixture.shaded.clone();
+ assert!(matches!(
+ process(&mut rendered, None, None),
+ ColorMode::Color
+ ));
+ let deviation = paper_deviation(&rendered, &fixture.regions);
+ eprintln!("colored stock: paper SD {deviation:.3}");
+ assert!(deviation < paper_deviation(&fixture.shaded, &fixture.regions) * 0.25);
+ let pixel = &rendered.data[3 * (40 * 416 + 40)..][..3];
+ assert!(pixel[2] > pixel[1] + 17);
+ assert!(pixel[1] > pixel[0] + 38);
+ let decode = &transfer_tables().decode;
+ let expected = chromaticity([decode[235], decode[211], decode[159]]);
+ let actual = chromaticity([
+ decode[pixel[2] as usize],
+ decode[pixel[1] as usize],
+ decode[pixel[0] as usize],
+ ]);
+ assert!(chroma_distance(expected, actual) < 0.005);
+ }
+
+ #[test]
+ fn keeps_tiny_coherent_stamps_and_one_pixel_color_strokes() {
+ let mut stamped = image(1600, 2200, [244; 3]);
+ for y in 1200..1209 {
+ for x in 400..407 {
+ let offset = (y * 1600 + x) * 3;
+ stamped.data[offset..offset + 3].copy_from_slice(&[61, 32, 194]);
+ }
+ }
+ assert!(matches!(
+ process(&mut stamped, None, None),
+ ColorMode::Color
+ ));
+ let mut signature = image(700, 1000, [244; 3]);
+ for x in 230..249 {
+ let offset = (720 * 700 + x) * 3;
+ signature.data[offset..offset + 3].copy_from_slice(&[120, 60, 44]);
+ }
+ assert!(matches!(
+ process(&mut signature, None, None),
+ ColorMode::Color
+ ));
+ }
+
+ #[test]
+ fn photograph_without_broad_paper_support_is_unchanged() {
+ let mut photo = image(384, 512, [0; 3]);
+ for y in 0..512 {
+ for x in 0..384 {
+ let index = (y * 384 + x) * 3;
+ let noise = ((x * 37 + y * 73 + x * y * 7) % 25) as u8;
+ let rgb = if y < 200 {
+ [65 + noise, 130 + noise, 190 + noise]
+ } else {
+ [45 + noise, 68 + noise, 29 + noise]
+ };
+ photo.data[index..index + 3].copy_from_slice(&[rgb[2], rgb[1], rgb[0]]);
+ }
+ }
+ let original = photo.data.clone();
+ assert!(matches!(process(&mut photo, None, None), ColorMode::Color));
+ assert_eq!(photo.data, original);
+ }
+
+ #[test]
+ fn large_neutral_graphics_are_not_used_as_dark_paper() {
+ let mut fixture = document(512, 704, [248; 3], false);
+ let transfer = transfer_tables();
+ for y in 300..530 {
+ for x in 150..380 {
+ let index = y * 512 + x;
+ let illumination =
+ 0.37 + 0.55 * x as f32 / 512.0 + 0.045 * (y as f32 / 704.0 * 5.0).sin();
+ fixture.clean.data[index * 3..index * 3 + 3].fill(119);
+ fixture.shaded.data[index * 3..index * 3 + 3].fill(encode(
+ transfer.decode[119] * illumination,
+ &transfer.encode,
+ ));
+ fixture.regions[index] = Region::Illustration;
+ }
+ }
+ let mut rendered = fixture.shaded;
+ process(&mut rendered, None, Some(ColorMode::Color));
+ let error = region_error(
+ &rendered,
+ &fixture.clean,
+ &fixture.regions,
+ Region::Illustration,
+ );
+ eprintln!("neutral illustration MAE {error:.3}");
+ assert!(region_mean(&rendered, &fixture.regions, Region::Illustration) < 132.0);
+ assert!(error < 16.0);
+ }
+
+ #[test]
+ fn missing_pixels_do_not_contribute_to_analysis_and_finish_neutral() {
+ let fixture = document(256, 352, [248; 3], false);
+ let mut first = fixture.shaded;
+ let mut second = first.clone();
+ let mut mask = vec![1; 256 * 352];
+ for y in 0..352 {
+ for x in 0..32 {
+ let index = y * 256 + x;
+ mask[index] = 0;
+ first.data[index * 3..index * 3 + 3].copy_from_slice(&[255, 0, 128]);
+ second.data[index * 3..index * 3 + 3].fill(0);
+ }
+ }
+ process(&mut first, Some(&mask), None);
+ process(&mut second, Some(&mask), None);
+ assert_eq!(first.data, second.data);
+ for (pixel, valid) in first.data.as_chunks::<3>().0.iter().zip(mask) {
+ if valid == 0 {
+ assert_eq!(*pixel, [255; 3]);
+ }
+ }
+ }
+
+ #[test]
+ fn invalid_inputs_are_rejected_before_mutation() {
+ let mut invalid = image(20, 20, [130; 3]);
+ let original = invalid.clone();
+ assert!(render_document(&mut invalid, Some(&[1; 7]), None).is_err());
+ assert_eq!(invalid, original);
+ invalid.width = i32::MAX;
+ invalid.height = i32::MAX;
+ let original = invalid.clone();
+ assert!(render_document(&mut invalid, None, None).is_err());
+ assert_eq!(invalid, original);
+ invalid.width = 0;
+ assert!(render_document(&mut invalid, None, None).is_err());
+ }
+
+ #[test]
+ fn blank_degenerate_and_fully_masked_pages_are_safe() {
+ for (width, height) in [(1, 1), (1, 400), (400, 1), (64, 96)] {
+ for value in [0, 71, 176, 245, 255] {
+ let mut blank = image(width, height, [value; 3]);
+ assert!(matches!(
+ process(&mut blank, None, None),
+ ColorMode::Grayscale
+ ));
+ assert!(
+ blank
+ .data
+ .as_chunks::<3>()
+ .0
+ .iter()
+ .all(|pixel| pixel[0] == pixel[1] && pixel[1] == pixel[2])
+ );
+ }
+ }
+ let mut missing = image(32, 48, [97, 124, 48]);
+ assert!(matches!(
+ process(&mut missing, Some(&[0; 32 * 48]), None),
+ ColorMode::Grayscale
+ ));
+ assert!(missing.data.iter().all(|&value| value == 255));
+ for (width, height) in [(1, 1), (1, 2), (2, 1), (4, 4)] {
+ let mut tiny = image(width, height, [184, 38, 91]);
+ assert!(matches!(process(&mut tiny, None, None), ColorMode::Color));
+ }
+ }
+
+ #[test]
+ fn noise_does_not_erase_faint_paper_marks() {
+ let mut fixture = document(384, 544, [248; 3], false);
+ for (index, pixel) in fixture
+ .shaded
+ .data
+ .as_chunks_mut::<3>()
+ .0
+ .iter_mut()
+ .enumerate()
+ {
+ let noise = ((index * 37 + index / 384 * 19) % 5) as i16 - 2;
+ pixel
+ .iter_mut()
+ .for_each(|value| *value = (*value as i16 + noise).clamp(0, 255) as u8);
+ }
+ let mut output = fixture.shaded;
+ process(&mut output, None, None);
+ let faint_contrast = region_mean(&output, &fixture.regions, Region::Paper)
+ - region_mean(&output, &fixture.regions, Region::Faint);
+ assert!(faint_contrast > 10.0);
+ assert!(paper_deviation(&output, &fixture.regions) < 5.0);
+ }
+
+ #[test]
+ fn color_conversion_is_identity_without_a_field() {
+ let mut pixels = image(256, 1, [0; 3]);
+ for value in 0..256 {
+ pixels.data[value * 3..value * 3 + 3].fill(value as u8);
+ }
+ let expected = pixels.data.clone();
+ render(&mut pixels, None, None, true);
+ assert_eq!(pixels.data, expected);
+ }
+
+ #[test]
+ fn broad_faint_graphics_keep_their_contrast() {
+ for value in [226, 236, 242] {
+ let mut page = image(512, 704, [248; 3]);
+ for y in 290..490 {
+ for x in 140..350 {
+ page.data[(y * 512 + x) * 3..][..3].fill(value);
+ }
+ }
+ process(&mut page, None, None);
+ let paper = page.data[(250 * 512 + 245) * 3];
+ let graphic = page.data[(390 * 512 + 245) * 3];
+ eprintln!("faint graphic {value}: paper {paper}, graphic {graphic}");
+ assert!(paper as i16 - graphic as i16 >= (248 - value as i16) * 3 / 4);
+ }
+ }
+
+ #[test]
+ fn embedded_photograph_receives_the_surrounding_paper_correction() {
+ let mut fixture = document(448, 640, [248; 3], true);
+ let transfer = transfer_tables();
+ for y in 130..510 {
+ for x in 90..360 {
+ let index = y * 448 + x;
+ let nx = x as f32 / 448.0;
+ let ny = y as f32 / 640.0;
+ let wave = (x as f32 * 0.073).sin() * (y as f32 * 0.052).cos();
+ let rgb = [0.15 + 0.05 * wave, 0.35 + 0.13 * wave, 0.48 - 0.07 * wave];
+ let illumination = 0.37 + 0.55 * nx + 0.045 * (ny * 5.0).sin();
+ for (channel, value) in rgb.into_iter().enumerate() {
+ fixture.clean.data[index * 3 + 2 - channel] = encode(value, &transfer.encode);
+ fixture.shaded.data[index * 3 + 2 - channel] =
+ encode(value * illumination, &transfer.encode);
+ }
+ fixture.regions[index] = Region::Illustration;
+ }
+ }
+ let mut output = fixture.shaded;
+ process(&mut output, None, None);
+ let error = region_error(
+ &output,
+ &fixture.clean,
+ &fixture.regions,
+ Region::Illustration,
+ );
+ eprintln!("embedded photograph MAE {error:.3}");
+ assert!(error < 8.0);
+ }
+
+ #[test]
+ fn held_out_illumination_shapes_and_neutral_casts_recover_the_reference() {
+ let transfer = transfer_tables();
+ for scenario in 0..3 {
+ let mut fixture = document(480, 688, [248; 3], true);
+ for (index, pixel) in fixture
+ .shaded
+ .data
+ .as_chunks_mut::<3>()
+ .0
+ .iter_mut()
+ .enumerate()
+ {
+ let x = (index % 480) as f32 / 480.0;
+ let y = (index / 480) as f32 / 688.0;
+ let illumination = match scenario {
+ 0 => 0.65 + 0.24 * (y * 5.1 + 0.8).cos(),
+ 1 => 0.91 - 0.47 * (-((x - 0.32).powi(2) + (y - 0.62).powi(2)) / 0.09).exp(),
+ _ => 0.36 + 0.55 * (0.3 * x + 0.7 * y),
+ };
+ for (channel, value) in pixel.iter_mut().enumerate() {
+ let cast = if scenario == 2 {
+ [0.975, 1.0, 1.025][channel]
+ } else {
+ 1.0
+ };
+ *value = encode(
+ transfer.decode[fixture.clean.data[index * 3 + channel] as usize]
+ * illumination
+ * cast,
+ &transfer.encode,
+ );
+ }
+ }
+ let mut output = fixture.shaded;
+ process(&mut output, None, None);
+ let paper = region_error(&output, &fixture.clean, &fixture.regions, Region::Paper);
+ let color = region_error(&output, &fixture.clean, &fixture.regions, Region::Color);
+ let deviation = paper_deviation(&output, &fixture.regions);
+ eprintln!(
+ "held-out lighting {scenario}: paper MAE {paper:.3}, color MAE {color:.3}, paper SD {deviation:.3}"
+ );
+ assert!(paper < 7.0);
+ assert!(color < 10.0);
+ assert!(deviation < 5.0);
+ }
+ }
+
+ #[test]
+ fn hard_shadow_boundaries_are_not_erased_as_paper() {
+ let mut fixture = document(400, 560, [248; 3], true);
+ let transfer = transfer_tables();
+ for (index, pixel) in fixture
+ .shaded
+ .data
+ .as_chunks_mut::<3>()
+ .0
+ .iter_mut()
+ .enumerate()
+ {
+ let illumination = if index % 400 < 170 { 0.42 } else { 0.91 };
+ for (channel, value) in pixel.iter_mut().enumerate() {
+ *value = encode(
+ transfer.decode[fixture.clean.data[index * 3 + channel] as usize]
+ * illumination,
+ &transfer.encode,
+ );
+ }
+ }
+ let mut output = fixture.shaded;
+ process(&mut output, None, None);
+ let faint_contrast = region_mean(&output, &fixture.regions, Region::Paper)
+ - region_mean(&output, &fixture.regions, Region::Faint);
+ assert!(faint_contrast > 10.0);
+ assert!(region_mean(&output, &fixture.regions, Region::Ink) < 42.0);
+ assert!(meaningful_color(&output, None));
+ }
+
+ #[test]
+ fn ambiguous_strong_illumination_and_colored_stock_keep_their_tint() {
+ for tint in [[248, 214, 171], [188, 221, 248], [243, 220, 238]] {
+ let mut page = image(320, 448, tint);
+ process(&mut page, None, None);
+ let original = [tint[2], tint[1], tint[0]];
+ assert!(
+ page.data
+ .as_chunks::<3>()
+ .0
+ .iter()
+ .all(|pixel| pixel.iter().zip(original).all(|(&a, b)| a.abs_diff(b) <= 1))
+ );
+ assert!(meaningful_color(&page, None));
+ }
+ }
+
+ #[test]
+ fn monochrome_photographic_tones_do_not_become_white_paper() {
+ let mut photo = image(384, 512, [0; 3]);
+ let transfer = transfer_tables();
+ for (index, pixel) in photo.data.as_chunks_mut::<3>().0.iter_mut().enumerate() {
+ let x = (index % 384) as f32 / 384.0;
+ let y = (index / 384) as f32 / 512.0;
+ let tone = 0.27
+ + 0.16 * (x * 8.0 + y * 3.0).sin() * (y * 7.0).cos()
+ + 0.06 * (x * 18.0 - y * 14.0).cos();
+ pixel.fill(encode(tone, &transfer.encode));
+ }
+ let before = photo.data.clone();
+ process(&mut photo, None, None);
+ let change = photo
+ .data
+ .iter()
+ .zip(before)
+ .map(|(&a, b)| a.abs_diff(b) as f32)
+ .sum::<f32>()
+ / photo.data.len() as f32;
+ eprintln!("monochrome photograph change {change:.3}");
+ assert!(change < 3.0);
+ }
+
+ #[test]
+ fn clipped_reflections_do_not_set_the_paper_level() {
+ let mut page = image(320, 448, [235; 3]);
+ for y in 150..300 {
+ for x in 130..260 {
+ page.data[(y * 320 + x) * 3..][..3].fill(255);
+ }
+ }
+ let analysis = analyze(&page, None, ANALYSIS_LONG_EDGE);
+ assert!(
+ analysis
+ .samples
+ .iter()
+ .filter(|sample| sample.unclipped == 0.0)
+ .all(|sample| sample.smoothness == 0.0)
+ );
+ process(&mut page, None, None);
+ let paper = page.data[(30 * 320 + 30) * 3];
+ assert!((250..=252).contains(&paper));
+ assert_eq!(page.data[(200 * 320 + 200) * 3], 255);
+ }
+
+ #[test]
+ #[ignore = "appearance calibration and timing report"]
+ fn analysis_resolution_calibration() {
+ for (width, height) in [(836, 1196), (1180, 1694), (1668, 2396)] {
+ let fixture = document(width, height, [248; 3], true);
+ for long_edge in [192, 288, 384, 512] {
+ let mut elapsed = Vec::new();
+ let mut output = fixture.shaded.clone();
+ let mut samples = 0;
+ for _ in 0..11 {
+ output.data.copy_from_slice(&fixture.shaded.data);
+ let start = std::time::Instant::now();
+ let analysis = analyze(&output, None, long_edge);
+ samples = analysis.samples.len();
+ let field = estimate_field(&analysis);
+ render(&mut output, None, field.as_ref(), false);
+ assert!(meaningful_color(&output, None));
+ elapsed.push(start.elapsed().as_secs_f64() * 1000.0);
+ }
+ elapsed.sort_unstable_by(f64::total_cmp);
+ let paper = region_error(&output, &fixture.clean, &fixture.regions, Region::Paper);
+ let color = region_error(&output, &fixture.clean, &fixture.regions, Region::Color);
+ let deviation = paper_deviation(&output, &fixture.regions);
+ eprintln!(
+ "{} pixels, analysis {long_edge}, sample bytes {}, median {:.3} ms, p95 {:.3} ms, paper MAE {paper:.3}, color MAE {color:.3}, paper SD {deviation:.3}",
+ width * height,
+ samples * std::mem::size_of::<Sample>(),
+ elapsed[5],
+ elapsed[10]
+ );
+ }
+ }
+ }
+}
diff --git a/rust/crates/ml/src/scan/boundary.rs b/rust/crates/ml/src/scan/boundary.rs
new file mode 100644
index 0000000000..5e5afb356c
--- /dev/null
+++ b/rust/crates/ml/src/scan/boundary.rs
@@ -0,0 +1,794 @@
+use super::page::{SourceExtent, area, cross, distance, points, quad_from_points, validate_quad};
+use super::segmentation::ProbabilityMap;
+use super::{OpResult, Point, Quad};
+use crate::cv::image::ImageU8;
+
+#[derive(Clone, Copy)]
+pub(super) enum SearchBudget {
+ Live,
+ Capture,
+}
+
+#[derive(Clone, Copy)]
+pub(super) struct MaskQuad {
+ pub corners: Quad,
+ extent: SourceExtent,
+}
+
+impl MaskQuad {
+ pub fn in_source(self, source: SourceExtent) -> Quad {
+ quad_from_points(points(self.corners).map(|p| Point {
+ x: p.x * source.width / self.extent.width,
+ y: p.y * source.height / self.extent.height,
+ }))
+ }
+}
+
+pub(super) struct Detection {
+ pub quad: Option<MaskQuad>,
+ pub confidence: f64,
+ pub reason: &'static str,
+}
+
+#[derive(Clone, Copy)]
+struct Line {
+ nx: f64,
+ ny: f64,
+ offset: f64,
+}
+
+impl Line {
+ fn through(a: Point, b: Point) -> Self {
+ let length = distance(a, b);
+ let nx = (b.y - a.y) / length;
+ let ny = (a.x - b.x) / length;
+ Self {
+ nx,
+ ny,
+ offset: nx * a.x + ny * a.y,
+ }
+ }
+
+ fn residual(self, p: Point) -> f64 {
+ self.nx * p.x + self.ny * p.y - self.offset
+ }
+
+ fn intersection(self, other: Self) -> Option<Point> {
+ let determinant = self.nx * other.ny - self.ny * other.nx;
+ if determinant.abs() < 1e-4 {
+ return None;
+ }
+ Some(Point {
+ x: (self.offset * other.ny - self.ny * other.offset) / determinant,
+ y: (self.nx * other.offset - self.offset * other.nx) / determinant,
+ })
+ }
+}
+
+fn fit_line(samples: &[Point], initial: Line, scale: f64) -> Line {
+ let mut line = initial;
+ for _ in 0..3 {
+ let mut sx = 0.0;
+ let mut sy = 0.0;
+ let mut weight = 0.0;
+ for &p in samples {
+ let w = (1.0 + (line.residual(p) / scale).powi(2)).recip();
+ sx += p.x * w;
+ sy += p.y * w;
+ weight += w;
+ }
+ if weight < 2.0 {
+ return initial;
+ }
+ let center = Point {
+ x: sx / weight,
+ y: sy / weight,
+ };
+ let mut xx = 0.0;
+ let mut yy = 0.0;
+ let mut xy = 0.0;
+ for &p in samples {
+ let w = (1.0 + (line.residual(p) / scale).powi(2)).recip();
+ xx += (p.x - center.x).powi(2) * w;
+ yy += (p.y - center.y).powi(2) * w;
+ xy += (p.x - center.x) * (p.y - center.y) * w;
+ }
+ let angle = 0.5 * (2.0 * xy).atan2(xx - yy);
+ let nx = -angle.sin();
+ let ny = angle.cos();
+ line = Line {
+ nx,
+ ny,
+ offset: nx * center.x + ny * center.y,
+ };
+ }
+ line
+}
+
+fn convex_hull(mut boundary: Vec<Point>) -> Vec<Point> {
+ boundary.sort_by(|a, b| a.x.total_cmp(&b.x).then(a.y.total_cmp(&b.y)));
+ boundary.dedup_by(|a, b| a.x == b.x && a.y == b.y);
+ let mut hull: Vec<Point> = Vec::new();
+ for &p in &boundary {
+ while hull.len() >= 2 && cross(hull[hull.len() - 2], hull[hull.len() - 1], p) <= 0.0 {
+ hull.pop();
+ }
+ hull.push(p);
+ }
+ let lower = hull.len();
+ for &p in boundary.iter().rev().skip(1) {
+ while hull.len() > lower && cross(hull[hull.len() - 2], hull[hull.len() - 1], p) <= 0.0 {
+ hull.pop();
+ }
+ hull.push(p);
+ }
+ hull.pop();
+ hull
+}
+
+fn simplify(mut hull: Vec<Point>, count: usize) -> Vec<Point> {
+ while hull.len() > count {
+ let Some(index) = (0..hull.len()).min_by(|&a, &b| {
+ let loss = |i| {
+ cross(
+ hull[(i + hull.len() - 1) % hull.len()],
+ hull[i],
+ hull[(i + 1) % hull.len()],
+ )
+ .abs()
+ };
+ loss(a).total_cmp(&loss(b)).then(a.cmp(&b))
+ }) else {
+ break;
+ };
+ hull.remove(index);
+ }
+ hull
+}
+
+fn canonical(mut p: [Point; 4], extent: SourceExtent) -> Quad {
+ if area(&p) < 0.0 {
+ p.reverse();
+ }
+ let first = (0..4)
+ .min_by(|&a, &b| {
+ let pa = p[a];
+ let pb = p[b];
+ (pa.x / extent.width + pa.y / extent.height)
+ .total_cmp(&(pb.x / extent.width + pb.y / extent.height))
+ .then(pa.x.total_cmp(&pb.x))
+ .then(pa.y.total_cmp(&pb.y))
+ })
+ .unwrap_or(0);
+ quad_from_points(std::array::from_fn(|i| p[(i + first) % 4]))
+}
+
+struct Component {
+ pixels: Vec<usize>,
+ boundary: Vec<Point>,
+}
+
+fn components(map: &ProbabilityMap, cutoff: f32, count: usize) -> Vec<Component> {
+ let mut visited = vec![false; map.values.len()];
+ let mut components = Vec::new();
+ for seed in 0..map.values.len() {
+ if visited[seed] || map.values[seed] < cutoff {
+ continue;
+ }
+ let mut queue = vec![seed];
+ visited[seed] = true;
+ let mut cursor = 0;
+ let mut boundary = Vec::new();
+ while cursor < queue.len() {
+ let index = queue[cursor];
+ cursor += 1;
+ let x = index % map.width;
+ let y = index / map.width;
+ for (dx, dy) in [(-1isize, 0isize), (1, 0), (0, -1), (0, 1)] {
+ let nx = x as isize + dx;
+ let ny = y as isize + dy;
+ if nx < 0 || ny < 0 || nx >= map.width as isize || ny >= map.height as isize {
+ boundary.push(Point {
+ x: x as f64 + 0.5 + dx as f64 * 0.5,
+ y: y as f64 + 0.5 + dy as f64 * 0.5,
+ });
+ continue;
+ }
+ let neighbor = ny as usize * map.width + nx as usize;
+ if map.values[neighbor] < cutoff {
+ let t = ((map.values[index] - cutoff)
+ / (map.values[index] - map.values[neighbor]))
+ as f64;
+ boundary.push(Point {
+ x: x as f64 + 0.5 + dx as f64 * t,
+ y: y as f64 + 0.5 + dy as f64 * t,
+ });
+ } else if !visited[neighbor] {
+ visited[neighbor] = true;
+ queue.push(neighbor);
+ }
+ }
+ }
+ if queue.len() >= 64 && queue.len() * 500 >= map.values.len() {
+ components.push(Component {
+ pixels: queue,
+ boundary,
+ });
+ }
+ }
+ components.sort_by(|a, b| {
+ b.pixels
+ .len()
+ .cmp(&a.pixels.len())
+ .then(a.pixels[0].cmp(&b.pixels[0]))
+ });
+ components.truncate(count);
+ components
+}
+
+fn supported_fit(p: [Point; 4], boundary: &[Point], extent: SourceExtent) -> Option<[Point; 4]> {
+ let initial = std::array::from_fn::<_, 4, _>(|i| Line::through(p[i], p[(i + 1) % 4]));
+ let mut sides: [Vec<Point>; 4] = std::array::from_fn(|_| Vec::new());
+ let band = extent.width.hypot(extent.height) * 0.025;
+ for &point in boundary {
+ let side = (0..4)
+ .min_by(|&a, &b| {
+ initial[a]
+ .residual(point)
+ .abs()
+ .total_cmp(&initial[b].residual(point).abs())
+ })
+ .unwrap_or(0);
+ if initial[side].residual(point).abs() < band {
+ sides[side].push(point);
+ }
+ }
+ let mut lines = initial;
+ for i in 0..4 {
+ if sides[i].len() < 4 {
+ return None;
+ }
+ lines[i] = fit_line(&sides[i], initial[i], (band / 4.0).max(0.5));
+ let along = |p: Point| lines[i].ny * p.x - lines[i].nx * p.y;
+ let lo = sides[i]
+ .iter()
+ .map(|&p| along(p))
+ .fold(f64::INFINITY, f64::min);
+ let hi = sides[i]
+ .iter()
+ .map(|&p| along(p))
+ .fold(f64::NEG_INFINITY, f64::max);
+ if hi - lo < distance(p[i], p[(i + 1) % 4]) * 0.55 {
+ return None;
+ }
+ }
+ let mut fitted = p;
+ for i in 0..4 {
+ fitted[i] = lines[(i + 3) % 4].intersection(lines[i])?;
+ }
+ if (0..4).any(|i| distance(fitted[i], p[i]) > band * 2.0) {
+ return None;
+ }
+ Some(fitted)
+}
+
+fn evidence(map: &ProbabilityMap, component: &Component, p: [Point; 4], live: bool) -> Option<f64> {
+ let extent = SourceExtent {
+ width: map.width as f64,
+ height: map.height as f64,
+ };
+ validate_quad(quad_from_points(p), extent).ok()?;
+ let lines = std::array::from_fn::<_, 4, _>(|i| Line::through(p[i], p[(i + 1) % 4]));
+ let inside = |q: Point| (0..4).all(|i| cross(p[i], p[(i + 1) % 4], q) >= -0.1);
+ let captured = component
+ .pixels
+ .iter()
+ .filter(|&&i| {
+ inside(Point {
+ x: (i % map.width) as f64 + 0.5,
+ y: (i / map.width) as f64 + 0.5,
+ })
+ })
+ .count() as f64
+ / component.pixels.len() as f64;
+ if captured < if live { 0.93 } else { 0.90 } {
+ return None;
+ }
+ let mut mass = 0.0;
+ let mut count = 0usize;
+ for y in (0..map.height).step_by(2) {
+ for x in (0..map.width).step_by(2) {
+ if inside(Point {
+ x: x as f64 + 0.5,
+ y: y as f64 + 0.5,
+ }) {
+ mass += map.values[y * map.width + x] as f64;
+ count += 1;
+ }
+ }
+ }
+ if count == 0 {
+ return None;
+ }
+ let purity = mass / count as f64;
+ if purity < if live { 0.78 } else { 0.68 } {
+ return None;
+ }
+ let mut residuals: Vec<f64> = component
+ .boundary
+ .iter()
+ .map(|&p| {
+ lines
+ .iter()
+ .map(|line| line.residual(p).abs())
+ .fold(f64::INFINITY, f64::min)
+ })
+ .collect();
+ residuals.sort_by(f64::total_cmp);
+ let residual = residuals[residuals.len() * 4 / 5];
+ let thickness =
+ (area(&p) / (0..4).map(|i| distance(p[i], p[(i + 1) % 4])).sum::<f64>()).max(1.0);
+ let tolerance = (thickness * if live { 0.065 } else { 0.085 }).max(1.2);
+ if residual > tolerance {
+ return None;
+ }
+ let fill = component.pixels.len() as f64 / area(&p);
+ if !(0.76..=1.16).contains(&fill) {
+ return None;
+ }
+ Some(
+ purity
+ * captured
+ * (1.0 - 0.15 * (residual / tolerance))
+ * (component.pixels.len() as f64 / map.values.len() as f64).sqrt(),
+ )
+}
+
+pub(super) fn locate(map: &ProbabilityMap, frame: SourceExtent, budget: SearchBudget) -> Detection {
+ let live = matches!(budget, SearchBudget::Live);
+ let extent = SourceExtent {
+ width: map.width as f64,
+ height: map.height as f64,
+ };
+ let mut best = Detection {
+ quad: None,
+ confidence: 0.0,
+ reason: "insufficient supported page boundary",
+ };
+ let cutoffs: &[f32] = if live { &[0.5] } else { &[0.5, 0.35, 0.7] };
+ for &cutoff in cutoffs {
+ for component in components(map, cutoff, if live { 2 } else { 4 }) {
+ let boundary_source: Vec<_> = component
+ .boundary
+ .iter()
+ .map(|p| Point {
+ x: p.x * frame.width / extent.width,
+ y: p.y * frame.height / extent.height,
+ })
+ .collect();
+ let hull_source = convex_hull(boundary_source);
+ if hull_source.len() < 4 {
+ continue;
+ }
+ let simplified = simplify(hull_source, if live { 4 } else { 6 });
+ for a in 0..simplified.len() - 3 {
+ for b in a + 1..simplified.len() - 2 {
+ for c in b + 1..simplified.len() - 1 {
+ for d in c + 1..simplified.len() {
+ let candidate =
+ [simplified[a], simplified[b], simplified[c], simplified[d]].map(
+ |p| Point {
+ x: p.x * extent.width / frame.width,
+ y: p.y * extent.height / frame.height,
+ },
+ );
+ let Some(candidate) =
+ supported_fit(candidate, &component.boundary, extent)
+ else {
+ continue;
+ };
+ let quad = canonical(candidate, extent);
+ let Some(confidence) = evidence(map, &component, points(quad), live)
+ else {
+ continue;
+ };
+ if confidence > best.confidence {
+ best = Detection {
+ quad: Some(MaskQuad {
+ corners: quad,
+ extent,
+ }),
+ confidence,
+ reason: "four supported sides",
+ };
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ best
+}
+
+fn brightness(source: &ImageU8, x: f64, y: f64) -> Option<f64> {
+ if x < 0.0 || y < 0.0 || x >= source.width as f64 || y >= source.height as f64 {
+ return None;
+ }
+ let offset = ((y as usize) * source.width as usize + x as usize) * 3;
+ Some(
+ (source.data[offset] as f64
+ + 2.0 * source.data[offset + 1] as f64
+ + source.data[offset + 2] as f64)
+ / 4.0,
+ )
+}
+
+pub(super) fn refine_capture(source: &ImageU8, quad: Quad) -> OpResult<Option<Quad>> {
+ let extent = SourceExtent::new(source.width, source.height)?;
+ validate_quad(quad, extent)?;
+ let scale = (640.0 / extent.width.max(extent.height)).min(1.0);
+ let preview = crate::cv::resize_u8(
+ source,
+ (extent.width * scale).round().max(1.0) as i32,
+ (extent.height * scale).round().max(1.0) as i32,
+ crate::cv::Interp::Area,
+ )?;
+ let sx = preview.width as f64 / extent.width;
+ let sy = preview.height as f64 / extent.height;
+ let p = points(quad).map(|p| Point {
+ x: p.x * sx,
+ y: p.y * sy,
+ });
+ let band = (preview.width.min(preview.height) as f64 * 0.008).clamp(1.5, 5.0);
+ let mut lines = std::array::from_fn::<_, 4, _>(|i| Line::through(p[i], p[(i + 1) % 4]));
+ let mut improved = 0;
+ let mut supported = 0;
+ for side in 0..4 {
+ let line = lines[side];
+ let edge_points = [p[side], p[(side + 1) % 4]];
+ let border = edge_points.iter().all(|p| p.x.abs() < 1.0)
+ || edge_points
+ .iter()
+ .all(|p| (p.x - preview.width as f64).abs() < 1.0)
+ || edge_points.iter().all(|p| p.y.abs() < 1.0)
+ || edge_points
+ .iter()
+ .all(|p| (p.y - preview.height as f64).abs() < 1.0);
+ if border {
+ supported += 1;
+ continue;
+ }
+ let mut samples = Vec::new();
+ let mut before = 0.0;
+ let mut after = 0.0;
+ for i in 0..48 {
+ let t = (i as f64 + 1.0) / 49.0;
+ let center = Point {
+ x: p[side].x + (p[(side + 1) % 4].x - p[side].x) * t,
+ y: p[side].y + (p[(side + 1) % 4].y - p[side].y) * t,
+ };
+ let gradient = |offset: f64| -> Option<f64> {
+ let a = brightness(
+ &preview,
+ center.x + line.nx * (offset - 1.25),
+ center.y + line.ny * (offset - 1.25),
+ )?;
+ let b = brightness(
+ &preview,
+ center.x + line.nx * (offset + 1.25),
+ center.y + line.ny * (offset + 1.25),
+ )?;
+ Some((a - b).abs())
+ };
+ let mut strongest = gradient(0.0).unwrap_or(0.0);
+ let base = strongest;
+ let mut offset = 0.0;
+ for step in -10..=10 {
+ let candidate = step as f64 * band / 10.0;
+ if let Some(value) = gradient(candidate) {
+ let penalized = value / (1.0 + 0.25 * (candidate / band).powi(2));
+ if penalized > strongest {
+ strongest = penalized;
+ offset = candidate;
+ }
+ }
+ }
+ if strongest >= 6.0 {
+ samples.push(Point {
+ x: center.x + line.nx * offset,
+ y: center.y + line.ny * offset,
+ });
+ before += base;
+ after += strongest;
+ }
+ }
+ if samples.len() < 32 {
+ continue;
+ }
+ let fitted = fit_line(&samples, line, 0.75);
+ if samples
+ .iter()
+ .filter(|&&p| fitted.residual(p).abs() < 1.0)
+ .count()
+ * 5
+ < samples.len() * 4
+ {
+ continue;
+ }
+ supported += 1;
+ if after <= before * 1.15 {
+ continue;
+ }
+ lines[side] = fitted;
+ improved += 1;
+ }
+ if supported < 3 {
+ log::debug!("capture rejected: only {supported} source-supported sides");
+ return Ok(None);
+ }
+ if improved == 0 {
+ return Ok(Some(quad));
+ }
+ let mut refined = p;
+ for i in 0..4 {
+ let Some(point) = lines[(i + 3) % 4].intersection(lines[i]) else {
+ return Ok(Some(quad));
+ };
+ if distance(point, p[i]) > band * 1.5 {
+ return Ok(Some(quad));
+ }
+ refined[i] = point;
+ }
+ let refined = quad_from_points(refined.map(|p| Point {
+ x: p.x / sx,
+ y: p.y / sy,
+ }));
+ if validate_quad(refined, extent).is_err() {
+ return Ok(Some(quad));
+ }
+ Ok(Some(refined))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn polygon_mask(p: [Point; 4]) -> OpResult<ProbabilityMap> {
+ let mut values = Vec::new();
+ for y in 0..256 {
+ for x in 0..256 {
+ let mut covered = 0;
+ for dy in [0.125, 0.375, 0.625, 0.875] {
+ for dx in [0.125, 0.375, 0.625, 0.875] {
+ let q = Point {
+ x: x as f64 + dx,
+ y: y as f64 + dy,
+ };
+ if (0..4).all(|i| cross(p[i], p[(i + 1) % 4], q) >= 0.0) {
+ covered += 1;
+ }
+ }
+ }
+ values.push(0.02 + 0.96 * covered as f32 / 16.0);
+ }
+ }
+ ProbabilityMap::new(values, 256, 256)
+ }
+
+ #[test]
+ fn independent_convex_page_fixtures_recover_all_corners() -> OpResult<()> {
+ let cases = [
+ [(30.0, 20.0), (225.0, 20.0), (225.0, 235.0), (30.0, 235.0)],
+ [(63.0, 40.0), (216.0, 17.0), (234.0, 212.0), (20.0, 238.0)],
+ [(100.0, 8.0), (153.0, 14.0), (159.0, 240.0), (90.0, 246.0)],
+ [(0.0, 20.0), (228.0, 30.0), (223.0, 238.0), (0.0, 246.0)],
+ [(0.0, 0.0), (256.0, 0.0), (256.0, 256.0), (0.0, 256.0)],
+ ];
+ for coordinates in cases {
+ let p = coordinates.map(|(x, y)| Point { x, y });
+ let map = polygon_mask(p)?;
+ for budget in [SearchBudget::Live, SearchBudget::Capture] {
+ let found = locate(&map, SourceExtent::new(1200, 800)?, budget)
+ .quad
+ .ok_or("page rejected")?;
+ for (actual, expected) in points(found.corners).into_iter().zip(p) {
+ assert!(
+ distance(actual, expected) < 2.0,
+ "{actual:?} versus {expected:?}"
+ );
+ }
+ }
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn noise_and_round_objects_do_not_become_pages() -> OpResult<()> {
+ for round in [false, true] {
+ let values = (0..256 * 256)
+ .map(|i| {
+ let x = (i % 256) as f64 - 128.0;
+ let y = (i / 256) as f64 - 128.0;
+ if round && x * x + y * y < 90.0 * 90.0 {
+ 0.98
+ } else if !round && i % 113 == 0 {
+ 0.9
+ } else {
+ 0.04
+ }
+ })
+ .collect();
+ let map = ProbabilityMap::new(values, 256, 256)?;
+ assert!(
+ locate(&map, SourceExtent::new(1200, 800)?, SearchBudget::Capture)
+ .quad
+ .is_none()
+ );
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn separate_components_are_not_enclosed_together() -> OpResult<()> {
+ let mut map = polygon_mask(
+ [(12.0, 40.0), (100.0, 40.0), (100.0, 220.0), (12.0, 220.0)]
+ .map(|(x, y)| Point { x, y }),
+ )?;
+ for y in 70..180 {
+ for x in 150..240 {
+ map.values[y * 256 + x] = 0.98;
+ }
+ }
+ let found = locate(&map, SourceExtent::new(1200, 800)?, SearchBudget::Capture)
+ .quad
+ .ok_or("page rejected")?;
+ assert!(points(found.corners).iter().all(|p| p.x < 110.0));
+ Ok(())
+ }
+
+ #[test]
+ fn canonical_labels_survive_nonsquare_source_scaling() -> OpResult<()> {
+ let p = [(0.4, 0.02), (0.9, 0.4), (0.6, 0.95), (0.1, 0.4)].map(|(x, y)| Point { x, y });
+ let a = canonical(
+ p,
+ SourceExtent {
+ width: 1.0,
+ height: 1.0,
+ },
+ );
+ let b = canonical(
+ p.map(|p| Point {
+ x: p.x * 2000.0,
+ y: p.y * 500.0,
+ }),
+ SourceExtent::new(2000, 500)?,
+ );
+ for (a, b) in points(a).into_iter().zip(points(b)) {
+ assert!((a.x - b.x / 2000.0).abs() < 1e-12 && (a.y - b.y / 500.0).abs() < 1e-12);
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn source_refinement_follows_long_page_edge_without_jumping_to_text() -> OpResult<()> {
+ let mut source = ImageU8::new(400, 300, 3, vec![50; 400 * 300 * 3])?;
+ for y in 40..260 {
+ for x in 60..340 {
+ source.data[(y * 400 + x) * 3..(y * 400 + x + 1) * 3].fill(235);
+ }
+ }
+ for y in (55..250).step_by(12) {
+ for x in 80..320 {
+ source.data[(y * 400 + x) * 3..(y * 400 + x + 1) * 3].fill(30);
+ }
+ }
+ let before = quad_from_points(
+ [(58.0, 38.0), (342.0, 38.0), (342.0, 262.0), (58.0, 262.0)]
+ .map(|(x, y)| Point { x, y }),
+ );
+ let after = refine_capture(&source, before)?.ok_or("source boundary rejected")?;
+ let expected = [(60.0, 40.0), (340.0, 40.0), (340.0, 260.0), (60.0, 260.0)]
+ .map(|(x, y)| Point { x, y });
+ let before_error: f64 = points(before)
+ .into_iter()
+ .zip(expected)
+ .map(|(a, b)| distance(a, b))
+ .sum();
+ let after_error: f64 = points(after)
+ .into_iter()
+ .zip(expected)
+ .map(|(a, b)| distance(a, b))
+ .sum();
+ assert!(
+ after_error < before_error,
+ "before {before_error}, after {after_error}"
+ );
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod benchmarks {
+ use super::*;
+ use std::time::Instant;
+
+ #[test]
+ #[ignore = "release stage performance measurements"]
+ fn detection_without_inference() -> OpResult<()> {
+ let p = [(50.0, 20.0), (226.0, 48.0), (207.0, 230.0), (18.0, 210.0)]
+ .map(|(x, y)| Point { x, y });
+ let values = (0..256 * 256)
+ .map(|i| {
+ let point = Point {
+ x: (i % 256) as f64 + 0.5,
+ y: (i / 256) as f64 + 0.5,
+ };
+ if (0..4).all(|j| cross(p[j], p[(j + 1) % 4], point) >= 0.0) {
+ 0.98
+ } else {
+ 0.02
+ }
+ })
+ .collect();
+ let map = ProbabilityMap::new(values, 256, 256)?;
+ let extent = SourceExtent::new(4000, 3000)?;
+ for (name, budget) in [
+ ("live", SearchBudget::Live),
+ ("capture", SearchBudget::Capture),
+ ] {
+ let mut times = Vec::new();
+ for _ in 0..101 {
+ let start = Instant::now();
+ assert!(locate(&map, extent, budget).quad.is_some());
+ times.push(start.elapsed().as_secs_f64() * 1000.0);
+ }
+ let cold = times.remove(0);
+ times.sort_by(f64::total_cmp);
+ println!(
+ "detect {name}: cold_ms={cold:.3}, median_ms={:.3}, p95_ms={:.3}",
+ times[50], times[95]
+ );
+ }
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod source_evidence_tests {
+ use super::*;
+
+ #[test]
+ fn unsupported_interior_text_regions_fall_back_to_the_whole_image() -> OpResult<()> {
+ let mut source = ImageU8::new(400, 300, 3, vec![235; 400 * 300 * 3])?;
+ for y in (35..270).step_by(15) {
+ for x in 40..360 {
+ source.data[(y * 400 + x) * 3..(y * 400 + x + 1) * 3].fill(35);
+ }
+ }
+ let quad = quad_from_points(
+ [(70.0, 50.0), (315.0, 65.0), (295.0, 250.0), (60.0, 240.0)]
+ .map(|(x, y)| Point { x, y }),
+ );
+ assert!(refine_capture(&source, quad)?.is_none());
+ Ok(())
+ }
+
+ #[test]
+ fn low_contrast_pages_and_actual_image_borders_remain_supported() -> OpResult<()> {
+ let mut source = ImageU8::new(400, 300, 3, vec![200; 400 * 300 * 3])?;
+ for y in 40..260 {
+ for x in 60..340 {
+ source.data[(y * 400 + x) * 3..(y * 400 + x + 1) * 3].fill(208);
+ }
+ }
+ let quad = quad_from_points(
+ [(60.0, 40.0), (340.0, 40.0), (340.0, 260.0), (60.0, 260.0)]
+ .map(|(x, y)| Point { x, y }),
+ );
+ assert!(refine_capture(&source, quad)?.is_some());
+ let extent = SourceExtent::new(400, 300)?;
+ assert!(refine_capture(&source, extent.full_frame())?.is_some());
+ Ok(())
+ }
+}
diff --git a/rust/crates/ml/src/scan/codec.rs b/rust/crates/ml/src/scan/codec.rs
index 2f46ac5cab..8886a60099 100644
--- a/rust/crates/ml/src/scan/codec.rs
+++ b/rust/crates/ml/src/scan/codec.rs
@@ -19,22 +19,59 @@ pub(crate) fn decode_bgr(bytes: &[u8]) -> Result<ImageU8, ScanError> {
ImageU8::new(width, height, 3, bgr).map_err(ScanError::Codec)
}
-pub(crate) fn encode_jpeg(image: &ImageU8, quality: u8) -> Result<Vec<u8>, ScanError> {
+pub(crate) fn encode_jpeg(mut image: ImageU8, quality: u8) -> Result<Vec<u8>, ScanError> {
if image.channels != 3 {
return Err(ScanError::Codec(format!(
"cannot encode a {}-channel image",
image.channels
)));
}
- let mut rgb = image.data.clone();
- for px in rgb.as_chunks_mut::<3>().0 {
+ for px in image.data.as_chunks_mut::<3>().0 {
px.swap(0, 2);
}
encode_rgb(
- &rgb,
+ &image.data,
image.width as u32,
image.height as u32,
EncodedImageFormat::Jpeg { quality },
)
.map_err(|err| ScanError::Codec(err.to_string()))
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::scan::segmentation::prepare_input;
+
+ #[test]
+ fn exif_orientation_is_applied_before_model_preparation() -> Result<(), ScanError> {
+ let source = ImageU8::new(
+ 37,
+ 21,
+ 3,
+ (0..37 * 21 * 3)
+ .map(|i| ((i * 37 + i / 13 * 17) % 256) as u8)
+ .collect(),
+ )
+ .map_err(ScanError::Codec)?;
+ let jpeg = encode_jpeg(source, 90)?;
+ let upright = decode_bgr(&jpeg)?;
+ let tiff = [
+ b'E', b'x', b'i', b'f', 0, 0, b'M', b'M', 0, 42, 0, 0, 0, 8, 0, 1, 1, 18, 0, 3, 0, 0,
+ 0, 1, 0, 6, 0, 0, 0, 0, 0, 0,
+ ];
+ let mut rotated_jpeg = jpeg[..2].to_vec();
+ rotated_jpeg.extend([0xff, 0xe1, 0, (tiff.len() + 2) as u8]);
+ rotated_jpeg.extend(tiff);
+ rotated_jpeg.extend(&jpeg[2..]);
+ let decoded = decode_bgr(&rotated_jpeg)?;
+ assert_eq!((decoded.width, decoded.height), (21, 37));
+ let expected = crate::cv::rotate_u8(&upright, 90).map_err(ScanError::Pipeline)?;
+ assert_eq!(decoded, expected);
+ assert_eq!(
+ prepare_input(&decoded).map_err(ScanError::Pipeline)?,
+ prepare_input(&expected).map_err(ScanError::Pipeline)?
+ );
+ Ok(())
+ }
+}
diff --git a/rust/crates/ml/src/scan/color.rs b/rust/crates/ml/src/scan/color.rs
deleted file mode 100644
index 8961451387..0000000000
--- a/rust/crates/ml/src/scan/color.rs
+++ /dev/null
@@ -1,153 +0,0 @@
-use super::OpResult;
-use super::detection::resize_for_max_pixels;
-use super::geometry::{Quad, norm};
-use super::mask::Mask;
-use crate::cv;
-use crate::cv::image::{ImageF32, ImageU8};
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum ColorMode {
- Color,
- Grayscale,
-}
-
-const CHROMA_THRESHOLD: f64 = 17.5;
-const PROPORTION_THRESHOLD: f64 = 0.0003;
-const LUMINANCE_MIN: f64 = 40.0;
-const LUMINANCE_MAX: f64 = 180.0;
-
-pub(crate) fn auto_color_mode(img: &ImageU8, mask: &Mask, quad: &Quad) -> OpResult<ColorMode> {
- let resized_img = resize_for_max_pixels(img, 1024.0 * 768.0)?;
- let work_size = resized_img.size();
-
- let doc_mask = document_mask(mask, quad, img.size(), work_size)?;
- let white_balanced = apply_gray_world_to_document(&resized_img, &doc_mask)?;
-
- let lab = cv::bgr_to_lab(&white_balanced)?;
- let channels = cv::split_u8(&lab)?;
- let luminance = &channels[0];
- let a = &channels[1];
- let b = &channels[2];
-
- let chroma = chroma(a, b)?;
-
- let color_mask = chroma.map_to_u8(|v| {
- if v > CHROMA_THRESHOLD as f32 {
- 255.0
- } else {
- 0.0
- }
- });
-
- let luminance_mask = cv::in_range_u8(luminance, LUMINANCE_MIN, LUMINANCE_MAX)?;
-
- let tmp = cv::bitwise_and_u8(&color_mask, &luminance_mask)?;
- let restricted_mask = cv::bitwise_and_u8(&tmp, &doc_mask)?;
-
- let colored_pixels = cv::count_non_zero(&restricted_mask)?;
- let total_pixels = cv::count_non_zero(&doc_mask)?;
-
- if total_pixels == 0 {
- return Ok(ColorMode::Grayscale);
- }
- let proportion = colored_pixels as f64 / total_pixels as f64;
- Ok(if proportion > PROPORTION_THRESHOLD {
- ColorMode::Color
- } else {
- ColorMode::Grayscale
- })
-}
-
-fn chroma(a: &ImageU8, b: &ImageU8) -> OpResult<ImageF32> {
- let data = a
- .data
- .iter()
- .zip(b.data.iter())
- .map(|(&a, &b)| {
- let a = a as f32 - 128.0;
- let b = b as f32 - 128.0;
- (a * a + b * b).sqrt()
- })
- .collect();
- ImageF32::new(a.width, a.height, 1, data)
-}
-
-fn erode_border(mask: &ImageU8, quad: &Quad) -> OpResult<ImageU8> {
- let min_dim = quad
- .edges()
- .iter()
- .map(|&(from, to)| norm(from, to))
- .fold(f64::INFINITY, f64::min);
- let mut k = (min_dim * 0.02 + 0.5).floor() as i32;
- k = k.clamp(3, 15);
- if k % 2 == 0 {
- k += 1;
- }
-
- let kernel = cv::ellipse_kernel(k)?;
- cv::morphology_erode(mask, &kernel)
-}
-
-fn document_mask(
- mask: &Mask,
- quad: &Quad,
- orig_size: (i32, i32),
- work_size: (i32, i32),
-) -> OpResult<ImageU8> {
- let mask_image = mask.binary_image()?;
- let resized_mask = cv::resize_u8(&mask_image, work_size.0, work_size.1, cv::Interp::Area)?;
-
- let resized_quad = quad.scaled_to(
- orig_size.0 as f64,
- orig_size.1 as f64,
- work_size.0 as f64,
- work_size.1 as f64,
- );
-
- let eroded_mask = erode_border(&resized_mask, &resized_quad)?;
-
- let pts: Vec<(i32, i32)> = resized_quad
- .corners()
- .iter()
- .map(|p| (p.x as i32, p.y as i32))
- .collect();
- let quad_mask = cv::fill_poly(eroded_mask.width, eroded_mask.height, &pts, 255.0)?;
-
- cv::bitwise_and_u8(&eroded_mask, &quad_mask)
-}
-
-pub(crate) fn apply_gray_world_to_document(img: &ImageU8, doc_mask: &ImageU8) -> OpResult<ImageU8> {
- assert_eq!(img.channels, 3, "gray-world expects an 8UC3 image");
-
- let non_zero = cv::count_non_zero(doc_mask)?;
- if non_zero == 0 {
- return Ok(img.clone());
- }
-
- let mean = cv::mean_u8c3_masked(img, doc_mask)?;
- let eps = 1e-6;
- let mean_b = if mean[0] < eps { eps } else { mean[0] };
- let mean_g = if mean[1] < eps { eps } else { mean[1] };
- let mean_r = if mean[2] < eps { eps } else { mean[2] };
-
- let mean_gray = (mean_b + mean_g + mean_r) / 3.0;
- let scales = [mean_gray / mean_b, mean_gray / mean_g, mean_gray / mean_r];
-
- let mut scaled = img.clone();
- for pixel in scaled.data.as_chunks_mut::<3>().0 {
- for (channel, scale) in pixel.iter_mut().zip(scales) {
- *channel = cv::saturate_u8_f32((*channel as f64 * scale) as f32);
- }
- }
-
- cv::copy_to_masked(&scaled, img, doc_mask)
-}
-
-pub(crate) fn quad_to_image(quad: &Quad, mask: &Mask, image_size: (i32, i32)) -> Quad {
- quad.scaled_to(
- mask.width as f64,
- mask.height as f64,
- image_size.0 as f64,
- image_size.1 as f64,
- )
-}
diff --git a/rust/crates/ml/src/scan/contour_orientation.rs b/rust/crates/ml/src/scan/contour_orientation.rs
deleted file mode 100644
index 9305676c51..0000000000
--- a/rust/crates/ml/src/scan/contour_orientation.rs
+++ /dev/null
@@ -1,325 +0,0 @@
-use super::geometry::{Point, average};
-
-const SMOOTH_WINDOW: i32 = 5;
-const MIN_SIDE_LENGTH_RATIO: f64 = 0.02;
-
-#[derive(Clone, Copy, Debug)]
-struct Line {
- p: Point,
- d: Point,
-}
-
-#[derive(Clone, Copy, Debug)]
-struct ContourSegment {
- start: usize,
- end: usize,
- angle: f64,
- length: f64,
-}
-
-pub(crate) fn find_quad_from_contour_orientation(contour: &[Point]) -> Option<Vec<Point>> {
- let max_angle_var = 5.0f64.to_radians();
- let merge_angle = 7.0f64.to_radians();
-
- if contour.len() < 20 {
- return None;
- }
-
- let angles = compute_smoothed_angles(contour, SMOOTH_WINDOW);
-
- let perimeter: f64 = contour
- .windows(2)
- .map(|w| (w[1].x - w[0].x).hypot(w[1].y - w[0].y))
- .sum();
-
- let min_length = perimeter * MIN_SIDE_LENGTH_RATIO;
- let segments = extract_segments(contour, &angles, max_angle_var, min_length);
- let merged = merge_segments(&segments, merge_angle);
- let dominant = select_dominant_segments(&merged, 4, 25.0f64.to_radians());
-
- if dominant.len() != 4 {
- return None;
- }
-
- let lines: Vec<Line> = dominant
- .iter()
- .map(|s| {
- let points: Vec<Point> = if s.start < s.end {
- contour[s.start..s.end].to_vec()
- } else {
- let mut v = contour[s.start..].to_vec();
- v.extend_from_slice(&contour[..s.end]);
- v
- };
- fit_line(&points)
- })
- .collect();
-
- let mut corners = Vec::with_capacity(4);
- for i in 0..4 {
- corners.push(intersect_lines(lines[i], lines[(i + 1) % 4])?);
- }
- Some(corners)
-}
-
-fn normalize_angle(a: f64) -> f64 {
- let mut x = a;
- while x <= -std::f64::consts::PI {
- x += 2.0 * std::f64::consts::PI;
- }
- while x > std::f64::consts::PI {
- x -= 2.0 * std::f64::consts::PI;
- }
- x
-}
-
-fn angle_diff(a: f64, b: f64) -> f64 {
- normalize_angle(a - b).abs()
-}
-
-fn angle_mean(a: f64, b: f64) -> f64 {
- let x = a.cos() + b.cos();
- let y = a.sin() + b.sin();
- y.atan2(x)
-}
-
-fn fit_line(points: &[Point]) -> Line {
- let cx = average(points.iter().map(|p| p.x));
- let cy = average(points.iter().map(|p| p.y));
-
- let mut xx = 0.0;
- let mut xy = 0.0;
- let mut yy = 0.0;
- for p in points {
- let dx = p.x - cx;
- let dy = p.y - cy;
- xx += dx * dx;
- xy += dx * dy;
- yy += dy * dy;
- }
-
- let theta = 0.5 * (2.0 * xy).atan2(xx - yy);
- Line {
- p: Point::new(cx, cy),
- d: Point::new(theta.cos(), theta.sin()),
- }
-}
-
-fn intersect_lines(l1: Line, l2: Line) -> Option<Point> {
- let x1 = l1.p.x;
- let y1 = l1.p.y;
- let x2 = x1 + l1.d.x;
- let y2 = y1 + l1.d.y;
-
- let x3 = l2.p.x;
- let y3 = l2.p.y;
- let x4 = x3 + l2.d.x;
- let y4 = y3 + l2.d.y;
-
- let denom = (x1 - x2) * (y3 - y4) - (y1 - y2) * (x3 - x4);
- if denom.abs() < 1e-6 {
- return None;
- }
-
- let px = ((x1 * y2 - y1 * x2) * (x3 - x4) - (x1 - x2) * (x3 * y4 - y3 * x4)) / denom;
- let py = ((x1 * y2 - y1 * x2) * (y3 - y4) - (y1 - y2) * (x3 * y4 - y3 * x4)) / denom;
- Some(Point::new(px, py))
-}
-
-fn extract_segments(
- contour: &[Point],
- angles: &[f64],
- max_angle_var: f64,
- min_length: f64,
-) -> Vec<ContourSegment> {
- let n = contour.len();
- let mut result = Vec::new();
-
- let start_index = find_best_start_index(angles);
-
- let segment_length = |s: usize, e: usize| -> f64 {
- let mut len = 0.0;
- let mut i = s;
- while i != e {
- let j = (i + 1) % n;
- len += (contour[j].x - contour[i].x).hypot(contour[j].y - contour[i].y);
- i = j;
- }
- len
- };
-
- let mut start = start_index;
- let mut ref_angle = angles[start_index];
-
- let mut steps = 1usize;
- while steps <= n {
- let idx = (start_index + steps) % n;
- if steps < n && angle_diff(angles[idx], ref_angle) < max_angle_var {
- ref_angle = angle_mean(ref_angle, angles[idx]);
- } else {
- let len = segment_length(start, idx);
- if len >= min_length {
- result.push(ContourSegment {
- start,
- end: idx,
- angle: ref_angle,
- length: len,
- });
- }
- start = idx;
- ref_angle = angles[idx];
- }
- steps += 1;
- }
-
- result
-}
-
-fn find_best_start_index(angles: &[f64]) -> usize {
- let n = angles.len();
- let mut best_index = 0usize;
- let mut best_delta = 0.0;
- for i in 0..n {
- let j = (i + 1) % n;
- let d = angle_diff(angles[i], angles[j]);
- if d > best_delta {
- best_delta = d;
- best_index = j;
- }
- }
- best_index
-}
-
-fn compute_smoothed_angles(contour: &[Point], window: i32) -> Vec<f64> {
- let n = contour.len();
- let ni = n as i32;
-
- let mut angles = vec![0.0f64; n];
- for i in 0..n {
- let p0 = contour[(i + n - 1) % n];
- let p1 = contour[(i + 1) % n];
- angles[i] = (p1.y - p0.y).atan2(p1.x - p0.x);
- }
-
- let cos_a: Vec<f64> = angles.iter().map(|a| a.cos()).collect();
- let sin_a: Vec<f64> = angles.iter().map(|a| a.sin()).collect();
-
- let mut smooth = vec![0.0f64; n];
- let mut sx = 0.0;
- let mut sy = 0.0;
- for k in -window..=window {
- let idx = ((k + ni) % ni) as usize;
- sx += cos_a[idx];
- sy += sin_a[idx];
- }
- smooth[0] = sy.atan2(sx);
-
- for (i, out) in smooth.iter_mut().enumerate().skip(1) {
- let ii = i as i32;
- let out_idx = ((ii - window - 1 + ni) % ni) as usize;
- let in_idx = ((ii + window) % ni) as usize;
- sx -= cos_a[out_idx];
- sy -= sin_a[out_idx];
- sx += cos_a[in_idx];
- sy += sin_a[in_idx];
- *out = sy.atan2(sx);
- }
- smooth
-}
-
-fn merge_segments(segments: &[ContourSegment], angle_threshold: f64) -> Vec<ContourSegment> {
- if segments.is_empty() {
- return Vec::new();
- }
- if segments.len() <= 4 {
- return segments.to_vec();
- }
-
- let mut merged: Vec<ContourSegment> = Vec::new();
- let mut cur = segments[0];
- for p in &segments[1..] {
- if angle_diff(p.angle, cur.angle) < angle_threshold {
- cur = ContourSegment {
- start: cur.start,
- end: p.end,
- angle: angle_mean(cur.angle, p.angle),
- length: cur.length + p.length,
- };
- } else {
- merged.push(cur);
- cur = *p;
- }
- }
- merged.push(cur);
- merged
-}
-
-fn select_dominant_segments(
- segments: &[ContourSegment],
- max_count: usize,
- min_angle_separation: f64,
-) -> Vec<ContourSegment> {
- let mut sorted = segments.to_vec();
- sorted.sort_by(|a, b| b.length.total_cmp(&a.length));
-
- let mut selected: Vec<ContourSegment> = Vec::new();
- for p in &sorted {
- let too_close = selected
- .iter()
- .any(|s| angle_diff(p.angle, s.angle) < min_angle_separation);
- if !too_close {
- selected.push(*p);
- if selected.len() == max_count {
- break;
- }
- }
- }
-
- selected.sort_by_key(|s| s.start);
- selected
-}
-
-#[cfg(test)]
-mod tests {
- use super::find_quad_from_contour_orientation;
- use crate::scan::geometry::{Point, norm};
-
- #[test]
- fn contour_orientation_recovers_a_rotated_rectangle() {
- let corners = [
- Point::new(40.0, 30.0),
- Point::new(220.0, 50.0),
- Point::new(200.0, 190.0),
- Point::new(20.0, 170.0),
- ];
- let mut contour = Vec::new();
- for i in 0..4 {
- let a = corners[i];
- let b = corners[(i + 1) % 4];
- let steps = 60;
- for s in 0..steps {
- let t = s as f64 / steps as f64;
- contour.push(Point::new(a.x + (b.x - a.x) * t, a.y + (b.y - a.y) * t));
- }
- }
- let quad =
- find_quad_from_contour_orientation(&contour).expect("quad from contour orientation");
- assert_eq!(quad.len(), 4);
- for corner in corners {
- let best = quad
- .iter()
- .map(|p| norm(*p, corner))
- .fold(f64::INFINITY, f64::min);
- assert!(
- best < 1.0,
- "corner {corner:?} not recovered (min dist {best})"
- );
- }
- }
-
- #[test]
- fn contour_orientation_rejects_short_contours() {
- let contour: Vec<Point> = (0..19).map(|i| Point::new(i as f64, 0.0)).collect();
- assert!(find_quad_from_contour_orientation(&contour).is_none());
- }
-}
diff --git a/rust/crates/ml/src/scan/detection.rs b/rust/crates/ml/src/scan/detection.rs
deleted file mode 100644
index 0804ac8beb..0000000000
--- a/rust/crates/ml/src/scan/detection.rs
+++ /dev/null
@@ -1,243 +0,0 @@
-use super::OpResult;
-use super::color::ColorMode;
-use super::contour_orientation::find_quad_from_contour_orientation;
-use super::geometry::{ImageSize, Point, Quad, create_quad};
-use super::mask::Mask;
-use super::perspective::estimate_real_dimensions;
-use super::postprocess::enhance_captured_image;
-use super::quad_score::score_quad_against_probmap;
-use crate::cv;
-use crate::cv::image::{Contour, ImageU8};
-
-const THRESHOLDS: [f64; 7] = [0.5, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95];
-const LIVE_THRESHOLDS: [f64; 1] = [0.9];
-
-pub(crate) fn size_trunc(width: f64, height: f64) -> (i32, i32) {
- (width as i32, height as i32)
-}
-
-pub(crate) fn detect_document_quad(
- mask: &Mask,
- original_size: ImageSize,
- is_live_analysis: bool,
-) -> OpResult<Option<Quad>> {
- let prob_image = mask.prob_image()?;
- let thresholds: &[f64] = if is_live_analysis {
- &LIVE_THRESHOLDS
- } else {
- &THRESHOLDS
- };
-
- let mut vertices =
- find_quad_from_orientation_with_adaptive_threshold(&prob_image, original_size, thresholds)?;
-
- if vertices.is_none() && !is_live_analysis {
- // Fallback: min-area rectangle of the biggest contour.
- if let Some(biggest) = biggest_contour(&refine_mask(&mask.binary_image()?)?)? {
- vertices = min_area_rect(&biggest.points, mask.width, mask.height);
- }
- }
-
- Ok(match vertices {
- Some(v) if v.len() == 4 => Some(create_quad(&v)),
- _ => None,
- })
-}
-
-fn find_quad_from_orientation_with_adaptive_threshold(
- prob_image: &ImageU8,
- original_size: ImageSize,
- thresholds: &[f64],
-) -> OpResult<Option<Vec<Point>>> {
- let probmap_smooth = cv::gaussian_blur_u8(prob_image, 3)?;
-
- let kernel = cv::ellipse_kernel(5)?;
- let prob_float = prob_image.to_f32();
- let mut best_quad: Option<Vec<Point>> = None;
- let mut best_score = 0.0f64;
-
- for &thr in thresholds {
- let bin = cv::threshold_binary_u8(&probmap_smooth, thr * 255.0, 255.0)?;
- let closed = cv::morphology_close(&bin, &kernel)?;
-
- if let Some(quad) = find_quad_from_orientation(&closed, original_size, &kernel)?
- && is_valid_quad(&quad)
- {
- let score = score_quad_against_probmap(&quad, &prob_float, 0.02)?;
- if score > best_score {
- best_score = score;
- best_quad = Some(quad);
- }
- }
- }
-
- Ok(best_quad)
-}
-
-// Deliberately no upper bound: edge-touching documents put corners past the mask edge.
-pub(crate) fn is_valid_quad(quad: &[Point]) -> bool {
- quad.iter().all(|p| p.x >= 0.0 && p.y >= 0.0)
-}
-
-fn find_quad_from_orientation(
- closed: &ImageU8,
- original_size: ImageSize,
- kernel: &ImageU8,
-) -> OpResult<Option<Vec<Point>>> {
- let opened = cv::morphology_open(closed, kernel)?;
- let Some(contour) = biggest_contour(&opened)? else {
- return Ok(None);
- };
-
- let scale_x = original_size.width / opened.width as f64;
- let scale_y = original_size.height / opened.height as f64;
-
- let scaled: Vec<Point> = contour
- .points
- .iter()
- .map(|&(x, y)| Point::new(x as f64 * scale_x, y as f64 * scale_y))
- .collect();
-
- Ok(find_quad_from_contour_orientation(&scaled).map(|corners| {
- corners
- .iter()
- .map(|p| Point::new(p.x / scale_x, p.y / scale_y))
- .collect()
- }))
-}
-
-pub(crate) fn min_area_rect(points: &[(i32, i32)], width: i32, height: i32) -> Option<Vec<Point>> {
- if points.len() < 3 {
- return None;
- }
- let pts: Vec<imageproc::point::Point<i32>> = points
- .iter()
- .map(|&(x, y)| imageproc::point::Point::new(x, y))
- .collect();
- let rect = imageproc::geometry::min_area_rect(&pts);
- if imageproc::geometry::contour_area(&rect) == 0.0 {
- return None;
- }
- Some(
- rect.iter()
- .map(|p| {
- Point::new(
- (p.x as f64).clamp(0.0, width as f64 - 1.0),
- (p.y as f64).clamp(0.0, height as f64 - 1.0),
- )
- })
- .collect(),
- )
-}
-
-pub(crate) fn biggest_contour(refined: &ImageU8) -> OpResult<Option<Contour>> {
- let blurred = cv::gaussian_blur_u8(refined, 5)?;
- let edges = cv::canny(&blurred, 75.0, 200.0)?;
- let contours = cv::find_contours(&edges)?;
-
- let mut biggest: Option<Contour> = None;
- let mut max_area = 0.0f64;
- for contour in contours {
- let area = contour.area.abs();
- if area > max_area {
- max_area = area;
- biggest = Some(contour);
- }
- }
- Ok(biggest)
-}
-
-fn refine_mask(original: &ImageU8) -> OpResult<ImageU8> {
- let binary_mask = cv::threshold_binary_u8(original, 128.0, 255.0)?;
- let kernel = cv::ellipse_kernel(5)?;
- let closed = cv::morphology_close(&binary_mask, &kernel)?;
- cv::morphology_open(&closed, &kernel)
-}
-
-pub(crate) fn resize_for_max_pixels(img: &ImageU8, max_pixels: f64) -> OpResult<ImageU8> {
- let orig_pixels = img.width as i64 * img.height as i64;
- if orig_pixels as f64 <= max_pixels {
- return Ok(img.clone());
- }
- let scale = (max_pixels / orig_pixels as f64).sqrt();
- let (width, height) = size_trunc(img.width as f64 * scale, img.height as f64 * scale);
- cv::resize_u8(img, width, height, cv::Interp::Area)
-}
-
-pub(crate) fn extract_document(
- input: &ImageU8,
- quad: &Quad,
- rotation_degrees: i32,
- color_mode: ColorMode,
- max_pixels: f64,
-) -> OpResult<ImageU8> {
- let estimated = estimate_real_dimensions(quad, input.width, input.height);
- let (target_width, target_height) = estimated.to_pixel_dimensions(quad);
-
- let corners = quad.corners();
- let src_corners = [
- (corners[0].x, corners[0].y),
- (corners[1].x, corners[1].y),
- (corners[2].x, corners[2].y),
- (corners[3].x, corners[3].y),
- ];
- let dst_corners = [
- (0.0, 0.0),
- (target_width, 0.0),
- (target_width, target_height),
- (0.0, target_height),
- ];
-
- let (out_width, out_height) = size_trunc(target_width, target_height);
- let warped = cv::warp_perspective(input, src_corners, dst_corners, out_width, out_height)?;
-
- let resized = resize_for_max_pixels(&warped, max_pixels)?;
- let enhanced = enhance_captured_image(&resized, color_mode)?;
- cv::rotate_u8(&enhanced, rotation_degrees)
-}
-
-#[cfg(test)]
-mod tests {
- use super::min_area_rect;
-
- fn assert_close(a: f64, b: f64, eps: f64) {
- assert!((a - b).abs() <= eps, "expected {a} ~= {b} (eps {eps})");
- }
-
- #[test]
- fn min_area_rect_of_axis_aligned_box() {
- let polygon = vec![(10, 20), (60, 20), (60, 90), (10, 90), (35, 55)];
- let rect = min_area_rect(&polygon, 256, 256).expect("rect");
- assert_eq!(rect.len(), 4);
- let xs: Vec<f64> = rect.iter().map(|p| p.x).collect();
- let ys: Vec<f64> = rect.iter().map(|p| p.y).collect();
- assert_close(xs.iter().cloned().fold(f64::INFINITY, f64::min), 10.0, 1e-9);
- assert_close(
- xs.iter().cloned().fold(f64::NEG_INFINITY, f64::max),
- 60.0,
- 1e-9,
- );
- assert_close(ys.iter().cloned().fold(f64::INFINITY, f64::min), 20.0, 1e-9);
- assert_close(
- ys.iter().cloned().fold(f64::NEG_INFINITY, f64::max),
- 90.0,
- 1e-9,
- );
- }
-
- #[test]
- fn min_area_rect_clips_to_image_bounds() {
- let polygon = vec![(-30, -30), (500, -30), (500, 500), (-30, 500)];
- let rect = min_area_rect(&polygon, 256, 256).expect("rect");
- for p in &rect {
- assert!((0.0..=255.0).contains(&p.x), "x out of bounds: {p:?}");
- assert!((0.0..=255.0).contains(&p.y), "y out of bounds: {p:?}");
- }
- }
-
- #[test]
- fn min_area_rect_rejects_degenerate_input() {
- assert!(min_area_rect(&[(0, 0), (1, 1)], 256, 256).is_none());
- assert!(min_area_rect(&[(0, 0), (5, 5), (10, 10)], 256, 256).is_none());
- }
-}
diff --git a/rust/crates/ml/src/scan/geometry.rs b/rust/crates/ml/src/scan/geometry.rs
deleted file mode 100644
index 23a3ce7316..0000000000
--- a/rust/crates/ml/src/scan/geometry.rs
+++ /dev/null
@@ -1,179 +0,0 @@
-pub(crate) fn average(values: impl Iterator<Item = f64>) -> f64 {
- let mut sum = 0.0;
- let mut count = 0usize;
- for v in values {
- sum += v;
- count += 1;
- }
- if count == 0 {
- f64::NAN
- } else {
- sum / count as f64
- }
-}
-
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub struct Point {
- pub x: f64,
- pub y: f64,
-}
-
-impl Point {
- pub fn new(x: f64, y: f64) -> Self {
- Self { x, y }
- }
-
- pub(crate) fn scaled(&self, scale_x: f64, scale_y: f64) -> Point {
- Point::new(self.x * scale_x, self.y * scale_y)
- }
-}
-
-pub(crate) fn norm(p1: Point, p2: Point) -> f64 {
- (p2.x - p1.x).hypot(p2.y - p1.y)
-}
-
-#[derive(Clone, Copy, Debug)]
-pub(crate) struct ImageSize {
- pub width: f64,
- pub height: f64,
-}
-
-impl ImageSize {
- pub(crate) fn new(width: f64, height: f64) -> Self {
- Self { width, height }
- }
-}
-
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub struct Quad {
- pub top_left: Point,
- pub top_right: Point,
- pub bottom_right: Point,
- pub bottom_left: Point,
-}
-
-impl Quad {
- pub fn corners(&self) -> [Point; 4] {
- [
- self.top_left,
- self.top_right,
- self.bottom_right,
- self.bottom_left,
- ]
- }
-
- pub(crate) fn edges(&self) -> [(Point, Point); 4] {
- [
- (self.top_left, self.top_right),
- (self.top_right, self.bottom_right),
- (self.bottom_right, self.bottom_left),
- (self.bottom_left, self.top_left),
- ]
- }
-
- pub(crate) fn rotate90(&self, iterations: i32, image_size: ImageSize) -> Quad {
- let rotate = |p: Point| -> Point {
- match iterations % 4 {
- 1 => Point::new(image_size.height - p.y, p.x),
- 2 => Point::new(image_size.width - p.x, image_size.height - p.y),
- 3 => Point::new(p.y, image_size.width - p.x),
- _ => p,
- }
- };
- create_quad(&[
- rotate(self.top_left),
- rotate(self.top_right),
- rotate(self.bottom_right),
- rotate(self.bottom_left),
- ])
- }
-
- pub(crate) fn scaled_to(
- &self,
- from_width: f64,
- from_height: f64,
- to_width: f64,
- to_height: f64,
- ) -> Quad {
- let scale_x = to_width / from_width;
- let scale_y = to_height / from_height;
- Quad {
- top_left: self.top_left.scaled(scale_x, scale_y),
- top_right: self.top_right.scaled(scale_x, scale_y),
- bottom_right: self.bottom_right.scaled(scale_x, scale_y),
- bottom_left: self.bottom_left.scaled(scale_x, scale_y),
- }
- }
-}
-
-pub(crate) fn create_quad(vertices: &[Point]) -> Quad {
- assert_eq!(vertices.len(), 4, "create_quad requires exactly 4 vertices");
- let cx = average(vertices.iter().map(|p| p.x));
- let cy = average(vertices.iter().map(|p| p.y));
-
- let mut sorted = vertices.to_vec();
- sorted.sort_by(|a, b| {
- let aa = (a.y - cy).atan2(a.x - cx);
- let bb = (b.y - cy).atan2(b.x - cx);
- aa.total_cmp(&bb)
- });
-
- Quad {
- top_left: sorted[0],
- top_right: sorted[1],
- bottom_right: sorted[2],
- bottom_left: sorted[3],
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::{Point, Quad, create_quad};
-
- fn assert_close(a: f64, b: f64, eps: f64) {
- assert!((a - b).abs() <= eps, "expected {a} ~= {b} (eps {eps})");
- }
-
- #[test]
- fn create_quad_orders_corners_by_angle_from_centroid() {
- let vertices = vec![
- Point::new(100.0, 100.0),
- Point::new(0.0, 100.0),
- Point::new(0.0, 0.0),
- Point::new(100.0, 0.0),
- ];
- let quad = create_quad(&vertices);
- assert_eq!(quad.top_left, Point::new(0.0, 0.0));
- assert_eq!(quad.top_right, Point::new(100.0, 0.0));
- assert_eq!(quad.bottom_right, Point::new(100.0, 100.0));
- assert_eq!(quad.bottom_left, Point::new(0.0, 100.0));
- }
-
- #[test]
- fn create_quad_is_idempotent() {
- let quad = create_quad(&[
- Point::new(10.0, 12.0),
- Point::new(90.0, 8.0),
- Point::new(95.0, 70.0),
- Point::new(5.0, 74.0),
- ]);
- let again = create_quad(&quad.corners());
- assert_eq!(quad.top_left, again.top_left);
- assert_eq!(quad.top_right, again.top_right);
- assert_eq!(quad.bottom_right, again.bottom_right);
- assert_eq!(quad.bottom_left, again.bottom_left);
- }
-
- #[test]
- fn scaled_to_maps_mask_space_to_image_space() {
- let quad = Quad {
- top_left: Point::new(0.0, 0.0),
- top_right: Point::new(128.0, 0.0),
- bottom_right: Point::new(128.0, 256.0),
- bottom_left: Point::new(0.0, 256.0),
- };
- let scaled = quad.scaled_to(256.0, 256.0, 1024.0, 512.0);
- assert_close(scaled.top_right.x, 512.0, 1e-9);
- assert_close(scaled.bottom_right.y, 512.0, 1e-9);
- }
-}
diff --git a/rust/crates/ml/src/scan/mask.rs b/rust/crates/ml/src/scan/mask.rs
deleted file mode 100644
index 0bdf8160df..0000000000
--- a/rust/crates/ml/src/scan/mask.rs
+++ /dev/null
@@ -1,27 +0,0 @@
-use super::OpResult;
-use crate::cv;
-use crate::cv::image::ImageU8;
-
-pub(crate) struct Mask {
- pub width: i32,
- pub height: i32,
- prob: Vec<u8>,
-}
-
-impl Mask {
- pub(crate) fn from_probmap(prob: Vec<u8>, width: i32, height: i32) -> Self {
- Self {
- width,
- height,
- prob,
- }
- }
-
- pub(crate) fn prob_image(&self) -> OpResult<ImageU8> {
- ImageU8::new(self.width, self.height, 1, self.prob.clone())
- }
-
- pub(crate) fn binary_image(&self) -> OpResult<ImageU8> {
- cv::threshold_binary_u8(&self.prob_image()?, 127.0, 255.0)
- }
-}
diff --git a/rust/crates/ml/src/scan/mod.rs b/rust/crates/ml/src/scan/mod.rs
index 82da7c2115..419bc08408 100644
--- a/rust/crates/ml/src/scan/mod.rs
+++ b/rust/crates/ml/src/scan/mod.rs
@@ -1,18 +1,30 @@
+mod appearance;
+mod boundary;
mod codec;
-mod color;
-mod contour_orientation;
-mod detection;
-mod geometry;
-mod mask;
-mod perspective;
-mod postprocess;
-mod quad_score;
+mod page;
mod scanner;
mod segmentation;
mod yuv;
-pub use color::ColorMode;
-pub use geometry::{Point, Quad};
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ColorMode {
+ Color,
+ Grayscale,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct Point {
+ pub x: f64,
+ pub y: f64,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct Quad {
+ pub top_left: Point,
+ pub top_right: Point,
+ pub bottom_right: Point,
+ pub bottom_left: Point,
+}
pub use scanner::{ReprocessOptions, ScanError, ScanResult, ScannerSession};
pub use segmentation::MASK_SIDE;
pub use yuv::PlaneLayout;
diff --git a/rust/crates/ml/src/scan/page.rs b/rust/crates/ml/src/scan/page.rs
new file mode 100644
index 0000000000..c36d1c0422
--- /dev/null
+++ b/rust/crates/ml/src/scan/page.rs
@@ -0,0 +1,442 @@
+use super::{OpResult, Point, Quad};
+use crate::cv::image::ImageU8;
+use crate::cv::projective::{ProjectiveMap, WarpedImage, warp_document};
+
+#[derive(Clone, Copy, Debug)]
+pub(super) struct SourceExtent {
+ pub width: f64,
+ pub height: f64,
+}
+
+impl SourceExtent {
+ pub fn new(width: i32, height: i32) -> OpResult<Self> {
+ if width <= 0 || height <= 0 {
+ return Err("source dimensions must be positive".to_owned());
+ }
+ Ok(Self {
+ width: width as f64,
+ height: height as f64,
+ })
+ }
+
+ pub fn full_frame(self) -> Quad {
+ quad_from_points([
+ Point { x: 0.0, y: 0.0 },
+ Point {
+ x: self.width,
+ y: 0.0,
+ },
+ Point {
+ x: self.width,
+ y: self.height,
+ },
+ Point {
+ x: 0.0,
+ y: self.height,
+ },
+ ])
+ }
+}
+
+pub(super) fn points(quad: Quad) -> [Point; 4] {
+ [
+ quad.top_left,
+ quad.top_right,
+ quad.bottom_right,
+ quad.bottom_left,
+ ]
+}
+
+pub(super) fn quad_from_points(p: [Point; 4]) -> Quad {
+ Quad {
+ top_left: p[0],
+ top_right: p[1],
+ bottom_right: p[2],
+ bottom_left: p[3],
+ }
+}
+
+pub(super) fn cross(a: Point, b: Point, c: Point) -> f64 {
+ (b.x - a.x) * (c.y - a.y) - (b.y - a.y) * (c.x - a.x)
+}
+
+pub(super) fn distance(a: Point, b: Point) -> f64 {
+ (a.x - b.x).hypot(a.y - b.y)
+}
+
+pub(super) fn area(p: &[Point; 4]) -> f64 {
+ (0..4)
+ .map(|i| p[i].x * p[(i + 1) % 4].y - p[i].y * p[(i + 1) % 4].x)
+ .sum::<f64>()
+ * 0.5
+}
+
+pub(super) fn validate_quad(quad: Quad, extent: SourceExtent) -> OpResult<()> {
+ let p = points(quad).map(|p| Point {
+ x: p.x / extent.width,
+ y: p.y / extent.height,
+ });
+ if p.iter().any(|p| {
+ !p.x.is_finite()
+ || !p.y.is_finite()
+ || p.x < -0.08
+ || p.y < -0.08
+ || p.x > 1.08
+ || p.y > 1.08
+ }) {
+ return Err("quad is non-finite or extends too far outside the source".to_owned());
+ }
+ let polygon_area = area(&p);
+ if polygon_area <= 1e-10
+ || (0..4).any(|i| cross(p[i], p[(i + 1) % 4], p[(i + 2) % 4]) <= polygon_area * 1e-7)
+ {
+ return Err("quad must be a non-degenerate clockwise convex polygon".to_owned());
+ }
+ ProjectiveMap::from_unit_quad(p.map(|p| (p.x, p.y)))?;
+ Ok(())
+}
+
+pub(super) fn quarter_turns(degrees: i32) -> OpResult<usize> {
+ if degrees % 90 != 0 {
+ return Err("rotation must be a multiple of 90 degrees".to_owned());
+ }
+ Ok((degrees / 90).rem_euclid(4) as usize)
+}
+
+pub(super) fn rotate_mask_quad(quad: Quad, turns: usize, side: f64) -> Quad {
+ let rotated = points(quad).map(|p| match turns {
+ 0 => p,
+ 1 => Point {
+ x: side - p.y,
+ y: p.x,
+ },
+ 2 => Point {
+ x: side - p.x,
+ y: side - p.y,
+ },
+ _ => Point {
+ x: p.y,
+ y: side - p.x,
+ },
+ });
+ quad_from_points(std::array::from_fn(|i| rotated[(i + 4 - turns) % 4]))
+}
+
+#[derive(Debug, Clone, Copy)]
+pub(super) enum AspectEvidence {
+ OppositeEdges,
+ CameraBlend(f64),
+}
+
+pub(super) struct RenderPlan {
+ pub width: i32,
+ pub height: i32,
+ pub map: ProjectiveMap,
+ pub aspect_evidence: AspectEvidence,
+}
+
+fn aspect_ratio(quad: Quad, extent: SourceExtent) -> OpResult<(f64, AspectEvidence)> {
+ let p = points(quad);
+ let horizontal = (distance(p[0], p[1]) * distance(p[2], p[3])).sqrt();
+ let vertical = (distance(p[1], p[2]) * distance(p[3], p[0])).sqrt();
+ let edge_ratio = horizontal / vertical;
+ let normalized = p.map(|p| {
+ (
+ (p.x - extent.width * 0.5) / extent.width,
+ (p.y - extent.height * 0.5) / extent.width,
+ )
+ });
+ let h = ProjectiveMap::from_unit_quad(normalized)?.coefficients;
+ let denominator = h[6] * h[7];
+ let focal_squared = -(h[0] * h[1] + h[3] * h[4]) / denominator;
+ let diagonal = extent.width.hypot(extent.height) / extent.width;
+ if denominator.abs() < 1e-5 || !focal_squared.is_finite() || focal_squared <= 0.0 {
+ return Ok((edge_ratio, AspectEvidence::OppositeEdges));
+ }
+ let focal = focal_squared.sqrt() / diagonal;
+ let ratio = ((h[0] * h[0] + h[3] * h[3] + focal_squared * h[6] * h[6])
+ / (h[1] * h[1] + h[4] * h[4] + focal_squared * h[7] * h[7]))
+ .sqrt();
+ let focal_support =
+ ((focal - 0.35) / 0.35).clamp(0.0, 1.0) * ((4.0 - focal) / 2.0).clamp(0.0, 1.0);
+ let perspective_support = (h[6].abs().min(h[7].abs()) / 0.2).clamp(0.0, 1.0);
+ let agreement = (1.0 - (ratio / edge_ratio).ln().abs() / 1.8f64.ln()).clamp(0.0, 1.0);
+ let confidence = focal_support * perspective_support * agreement;
+ if confidence == 0.0 || !ratio.is_finite() {
+ return Ok((edge_ratio, AspectEvidence::OppositeEdges));
+ }
+ Ok((
+ (edge_ratio.ln() * (1.0 - confidence) + ratio.ln() * confidence).exp(),
+ AspectEvidence::CameraBlend(confidence),
+ ))
+}
+
+pub(super) fn checked_dimensions(width: f64, height: f64, max_pixels: u32) -> OpResult<(i32, i32)> {
+ if max_pixels == 0
+ || !width.is_finite()
+ || !height.is_finite()
+ || width <= 0.0
+ || height <= 0.0
+ || !(width * height).is_finite()
+ {
+ return Err("render dimensions and pixel budget must be positive and finite".to_owned());
+ }
+ let scale = (max_pixels as f64 / (width * height)).sqrt().min(1.0);
+ let mut w = (width * scale + f64::EPSILON * width * scale * 16.0)
+ .floor()
+ .max(1.0)
+ .min(max_pixels as f64) as u64;
+ let mut h = (height * scale + f64::EPSILON * height * scale * 16.0)
+ .floor()
+ .max(1.0)
+ .min(max_pixels as f64) as u64;
+ if w * h > max_pixels as u64 {
+ if w >= h {
+ w = max_pixels as u64 / h;
+ } else {
+ h = max_pixels as u64 / w;
+ }
+ }
+ let w = i32::try_from(w).map_err(|_| "render width is too large".to_owned())?;
+ let h = i32::try_from(h).map_err(|_| "render height is too large".to_owned())?;
+ Ok((w, h))
+}
+
+impl RenderPlan {
+ pub fn new(
+ quad: Quad,
+ extent: SourceExtent,
+ rotation_degrees: i32,
+ max_pixels: u32,
+ ) -> OpResult<Self> {
+ validate_quad(quad, extent)?;
+ let turns = quarter_turns(rotation_degrees)?;
+ let p = points(quad);
+ let (ratio, aspect_evidence) = aspect_ratio(quad, extent)?;
+ let horizontal = (distance(p[0], p[1]) * distance(p[2], p[3])).sqrt();
+ let vertical = (distance(p[1], p[2]) * distance(p[3], p[0])).sqrt();
+ let height = vertical.min(horizontal / ratio);
+ let (mut width, mut height) = checked_dimensions(height * ratio, height, max_pixels)?;
+ if turns % 2 == 1 {
+ std::mem::swap(&mut width, &mut height);
+ }
+ let corners = std::array::from_fn(|i| {
+ let p = p[(i + 4 - turns) % 4];
+ (p.x, p.y)
+ });
+ let map = ProjectiveMap::from_unit_quad(corners)?;
+ Ok(Self {
+ width,
+ height,
+ map,
+ aspect_evidence,
+ })
+ }
+
+ pub fn render(&self, source: &ImageU8) -> OpResult<WarpedImage> {
+ match self.aspect_evidence {
+ AspectEvidence::OppositeEdges => {
+ log::debug!("page proportions: opposite-edge estimate")
+ }
+ AspectEvidence::CameraBlend(confidence) => {
+ log::debug!("page proportions: camera blend {confidence:.3}")
+ }
+ }
+ warp_document(source, self.map, self.width, self.height)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn full_frame_and_quarter_turns_keep_every_edge_pixel() -> OpResult<()> {
+ let source = ImageU8::new(7, 5, 3, (0..105).map(|i| i as u8).collect())?;
+ let extent = SourceExtent::new(7, 5)?;
+ for turns in 0..4 {
+ let plan = RenderPlan::new(extent.full_frame(), extent, turns * 90, 35)?;
+ let output = plan.render(&source)?.image;
+ assert_eq!(output.width, if turns % 2 == 0 { 7 } else { 5 });
+ assert_eq!(output.height, if turns % 2 == 0 { 5 } else { 7 });
+ for y in 0..5 {
+ for x in 0..7 {
+ let (dx, dy) = match turns {
+ 0 => (x, y),
+ 1 => (4 - y, x),
+ 2 => (6 - x, 4 - y),
+ _ => (y, 6 - x),
+ };
+ let a = (y * 7 + x) * 3;
+ let b = (dy * output.width as usize + dx) * 3;
+ assert_eq!(
+ &source.data[a..a + 3],
+ &output.data[b..b + 3],
+ "turns={turns} x={x} y={y}"
+ );
+ }
+ }
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn tiny_budgets_and_extreme_receipts_stay_nonempty() -> OpResult<()> {
+ for budget in 1..100 {
+ for (w, h) in [(100.0, 3000.0), (8000.0, 30.0), (1.0, 1.0)] {
+ let (width, height) = checked_dimensions(w, h, budget)?;
+ assert!(width > 0 && height > 0);
+ assert!(width as u64 * height as u64 <= budget as u64);
+ }
+ }
+ assert!(checked_dimensions(10.0, 10.0, 0).is_err());
+ assert!(checked_dimensions(f64::INFINITY, 10.0, 10).is_err());
+ assert!(checked_dimensions(1e300, 1e300, 100).is_err());
+ Ok(())
+ }
+
+ #[test]
+ fn invalid_quads_fail_before_allocation() -> OpResult<()> {
+ let extent = SourceExtent::new(800, 600)?;
+ let valid = extent.full_frame();
+ let mut p = points(valid);
+ p.swap(1, 2);
+ assert!(RenderPlan::new(quad_from_points(p), extent, 0, 2_000_000).is_err());
+ let mut p = points(valid);
+ p[0].x = f64::NAN;
+ assert!(RenderPlan::new(quad_from_points(p), extent, 0, 2_000_000).is_err());
+ assert!(RenderPlan::new(valid, extent, 45, 2_000_000).is_err());
+ assert!(RenderPlan::new(valid, extent, -90, 2_000_000).is_ok());
+ assert!(
+ RenderPlan::new(
+ quad_from_points([Point { x: 1.0, y: 1.0 }; 4]),
+ extent,
+ 0,
+ 2_000_000
+ )
+ .is_err()
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn front_facing_cards_receipts_and_landscape_preserve_proportions() -> OpResult<()> {
+ for (width, height) in [(400, 700), (700, 400), (100, 1700), (1300, 120)] {
+ let extent = SourceExtent::new(width, height)?;
+ let plan = RenderPlan::new(extent.full_frame(), extent, 0, 2_000_000)?;
+ assert!(
+ ((plan.width as f64 / plan.height as f64) / (width as f64 / height as f64) - 1.0)
+ .abs()
+ < 0.005
+ );
+ }
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod benchmarks {
+ use super::*;
+ use std::time::Instant;
+
+ #[test]
+ #[ignore = "release performance and allocation measurements"]
+ fn render_source_and_output_budgets() -> OpResult<()> {
+ for (width, height) in [(4000, 3000), (6000, 4000), (8000, 6000)] {
+ let source = ImageU8::new(
+ width,
+ height,
+ 3,
+ (0..width as usize * height as usize * 3)
+ .map(|i| ((i * 17 + i / 31) % 256) as u8)
+ .collect(),
+ )?;
+ let extent = SourceExtent::new(width, height)?;
+ for budget in [1_000_000, 2_000_000, 4_000_000] {
+ let plan = RenderPlan::new(extent.full_frame(), extent, 0, budget)?;
+ let mut times = Vec::new();
+ for _ in 0..21 {
+ let start = Instant::now();
+ let rendered = plan.render(&source)?;
+ std::hint::black_box(rendered.image.data);
+ times.push(start.elapsed().as_secs_f64() * 1000.0);
+ }
+ let cold = times.remove(0);
+ times.sort_by(f64::total_cmp);
+ println!(
+ "render {width}x{height} -> {}x{}, cold_ms={cold:.3}, median_ms={:.3}, p95_ms={:.3}, decoded_source_bytes={}, output_bytes={}",
+ plan.width,
+ plan.height,
+ times[2],
+ times[4],
+ source.data.len(),
+ plan.width as usize * plan.height as usize * 3
+ );
+ }
+ }
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod camera_geometry_tests {
+ use super::*;
+
+ fn camera_quad(width: f64, height: f64, yaw: f64, pitch: f64) -> Quad {
+ let project = |x: f64, y: f64| {
+ let x_camera = yaw.cos() * x + yaw.sin() * pitch.sin() * y;
+ let y_camera = pitch.cos() * y;
+ let z_camera = 1400.0 - yaw.sin() * x + yaw.cos() * pitch.sin() * y;
+ Point {
+ x: 600.0 + 1100.0 * x_camera / z_camera,
+ y: 450.0 + 1100.0 * y_camera / z_camera,
+ }
+ };
+ quad_from_points([
+ project(-width / 2.0, -height / 2.0),
+ project(width / 2.0, -height / 2.0),
+ project(width / 2.0, height / 2.0),
+ project(-width / 2.0, height / 2.0),
+ ])
+ }
+
+ #[test]
+ fn camera_constraints_improve_supported_oblique_rectangles() -> OpResult<()> {
+ let extent = SourceExtent::new(1200, 900)?;
+ for (w, h, yaw, pitch) in [
+ (400.0, 560.0, 0.45, 0.35),
+ (400.0, 560.0, 0.8, 0.55),
+ (100.0, 900.0, 0.35, 0.3),
+ (600.0, 340.0, 0.65, 0.45),
+ ] {
+ let quad = camera_quad(w, h, yaw, pitch);
+ let p = points(quad);
+ let edge = ((distance(p[0], p[1]) * distance(p[2], p[3]))
+ / (distance(p[1], p[2]) * distance(p[3], p[0])))
+ .sqrt();
+ let (ratio, evidence) = aspect_ratio(quad, extent)?;
+ let error = (ratio / (w / h)).ln().abs();
+ let edge_error = (edge / (w / h)).ln().abs();
+ assert!(
+ error <= edge_error + 1e-10,
+ "{evidence:?}: {ratio} vs {}",
+ w / h
+ );
+ assert!(error < 0.22, "conservative aspect error {error}");
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn weak_perspective_proportions_change_continuously() -> OpResult<()> {
+ let extent = SourceExtent::new(1200, 900)?;
+ let mut previous = 400.0 / 560.0;
+ for step in 0..100 {
+ let angle = step as f64 / 1000.0;
+ let (ratio, _) = aspect_ratio(camera_quad(400.0, 560.0, angle, angle * 0.7), extent)?;
+ assert!((ratio - previous).abs() < 0.001);
+ previous = ratio;
+ }
+ Ok(())
+ }
+}
diff --git a/rust/crates/ml/src/scan/perspective.rs b/rust/crates/ml/src/scan/perspective.rs
deleted file mode 100644
index 94ab229406..0000000000
--- a/rust/crates/ml/src/scan/perspective.rs
+++ /dev/null
@@ -1,178 +0,0 @@
-use super::geometry::{Point, Quad, norm};
-
-#[derive(Clone, Copy, Debug)]
-struct Vector3D {
- x: f64,
- y: f64,
- z: f64,
-}
-
-impl Vector3D {
- fn new(x: f64, y: f64, z: f64) -> Self {
- Self { x, y, z }
- }
-
- fn sub(self, other: Vector3D) -> Vector3D {
- Vector3D::new(self.x - other.x, self.y - other.y, self.z - other.z)
- }
-
- fn scale(self, t: f64) -> Vector3D {
- Vector3D::new(self.x * t, self.y * t, self.z * t)
- }
-
- fn dot(self, other: Vector3D) -> f64 {
- self.x * other.x + self.y * other.y + self.z * other.z
- }
-
- fn cross(self, other: Vector3D) -> Vector3D {
- Vector3D::new(
- self.y * other.z - self.z * other.y,
- self.z * other.x - self.x * other.z,
- self.x * other.y - self.y * other.x,
- )
- }
-
- fn norm(self) -> f64 {
- (self.x * self.x + self.y * self.y + self.z * self.z).sqrt()
- }
-}
-
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub(crate) struct EstimatedDimensions {
- pub(crate) width: f64,
- pub(crate) height: f64,
-}
-
-impl EstimatedDimensions {
- pub(crate) fn to_pixel_dimensions(self, quad: &Quad) -> (f64, f64) {
- let w =
- (norm(quad.top_left, quad.top_right) + norm(quad.bottom_left, quad.bottom_right)) / 2.0;
- let h =
- (norm(quad.top_left, quad.bottom_left) + norm(quad.top_right, quad.bottom_right)) / 2.0;
- let projected_area = w * h;
-
- let ratio = self.height / self.width;
- let target_width = (projected_area / ratio).sqrt();
- let target_height = target_width * ratio;
- (target_width, target_height)
- }
-}
-
-pub(crate) fn estimate_real_dimensions(
- quad: &Quad,
- image_width: i32,
- image_height: i32,
-) -> EstimatedDimensions {
- let average_sides = || EstimatedDimensions {
- width: (norm(quad.top_left, quad.top_right) + norm(quad.bottom_left, quad.bottom_right))
- / 2.0,
- height: (norm(quad.top_left, quad.bottom_left) + norm(quad.top_right, quad.bottom_right))
- / 2.0,
- };
-
- let to_h = |p: Point| Vector3D::new(p.x, p.y, 1.0);
- let line_through = |p1: Point, p2: Point| to_h(p1).cross(to_h(p2));
-
- let v1h = line_through(quad.top_left, quad.top_right)
- .cross(line_through(quad.bottom_left, quad.bottom_right));
- let v2h = line_through(quad.top_left, quad.bottom_left)
- .cross(line_through(quad.top_right, quad.bottom_right));
-
- if v1h.z.abs() < 1e-6 || v2h.z.abs() < 1e-6 {
- return average_sides();
- }
-
- let cx = image_width as f64 / 2.0;
- let cy = image_height as f64 / 2.0;
-
- let v1 = Point::new(v1h.x / v1h.z - cx, v1h.y / v1h.z - cy);
- let v2 = Point::new(v2h.x / v2h.z - cx, v2h.y / v2h.z - cy);
-
- let f2 = -(v1.x * v2.x + v1.y * v2.y);
- if f2 <= 0.0 {
- return average_sides();
- }
- let f = f2.sqrt();
-
- if f > (image_width.max(image_height)) as f64 * 1.2 {
- return average_sides();
- }
-
- let d1 = Vector3D::new(v1.x, v1.y, f);
- let d2 = Vector3D::new(v2.x, v2.y, f);
- let n = d1.cross(d2);
-
- let ray = |p: Point| Vector3D::new((p.x - cx) / f, (p.y - cy) / f, 1.0);
-
- let corner3d = |p: Point| {
- let r = ray(p);
- r.scale(1.0 / n.dot(r))
- };
-
- let x_tl = corner3d(quad.top_left);
- let x_tr = corner3d(quad.top_right);
- let x_br = corner3d(quad.bottom_right);
- let x_bl = corner3d(quad.bottom_left);
-
- let real_w = (x_tr.sub(x_tl).norm() + x_br.sub(x_bl).norm()) / 2.0;
- let real_h = (x_bl.sub(x_tl).norm() + x_br.sub(x_tr).norm()) / 2.0;
-
- EstimatedDimensions {
- width: real_w,
- height: real_h,
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::{EstimatedDimensions, estimate_real_dimensions};
- use crate::scan::geometry::{Point, Quad};
-
- fn assert_close(a: f64, b: f64, eps: f64) {
- assert!((a - b).abs() <= eps, "expected {a} ~= {b} (eps {eps})");
- }
-
- fn rect_quad(w: f64, h: f64) -> Quad {
- Quad {
- top_left: Point::new(0.0, 0.0),
- top_right: Point::new(w, 0.0),
- bottom_right: Point::new(w, h),
- bottom_left: Point::new(0.0, h),
- }
- }
-
- #[test]
- fn estimate_real_dimensions_falls_back_for_parallel_sides() {
- let quad = rect_quad(400.0, 300.0);
- let dims = estimate_real_dimensions(&quad, 1000, 800);
- assert_eq!(
- dims,
- EstimatedDimensions {
- width: 400.0,
- height: 300.0
- }
- );
- }
-
- #[test]
- fn estimate_real_dimensions_falls_back_for_weak_perspective() {
- let quad = Quad {
- top_left: Point::new(100.0, 100.0),
- top_right: Point::new(900.0, 100.0),
- bottom_right: Point::new(899.0, 700.0),
- bottom_left: Point::new(101.0, 700.0),
- };
- let dims = estimate_real_dimensions(&quad, 1000, 800);
- assert_close(dims.width, 799.0, 1e-9);
- assert_close(dims.height, 600.0008333, 1e-6);
- }
-
- #[test]
- fn to_pixel_dimensions_preserves_area_and_ratio() {
- let quad = rect_quad(400.0, 300.0);
- let dims = estimate_real_dimensions(&quad, 1000, 800);
- let (w, h) = dims.to_pixel_dimensions(&quad);
- assert_close(w * h, 400.0 * 300.0, 1e-6);
- assert_close(h / w, 300.0 / 400.0, 1e-12);
- }
-}
diff --git a/rust/crates/ml/src/scan/postprocess.rs b/rust/crates/ml/src/scan/postprocess.rs
deleted file mode 100644
index b08113e8f2..0000000000
--- a/rust/crates/ml/src/scan/postprocess.rs
+++ /dev/null
@@ -1,305 +0,0 @@
-use super::OpResult;
-use super::color::ColorMode;
-use super::detection::size_trunc;
-use crate::cv;
-use crate::cv::image::{ImageF32, ImageU8};
-
-pub(crate) fn enhance_captured_image(img: &ImageU8, color_mode: ColorMode) -> OpResult<ImageU8> {
- match color_mode {
- ColorMode::Color => multi_scale_retinex_on_l(img),
- ColorMode::Grayscale => enhance_grayscale_image(img),
- }
-}
-
-fn clamp_255(v: f32) -> f32 {
- let v = if v < 255.0 { v } else { 255.0 };
- if v > 0.0 { v } else { 0.0 }
-}
-
-const SHADOW_MAP_LONG_EDGE: f64 = 256.0;
-const SHADOW_KERNEL_FRAC: f64 = 1.0 / 16.0;
-const SHADOW_MAX_GAIN: f32 = 2.8;
-const SHADOW_PAPER_PERCENTILE: f64 = 0.90;
-
-fn remove_shadows(channel: &ImageU8) -> OpResult<ImageU8> {
- let (width, height) = channel.size();
- let long_edge = width.max(height) as f64;
- let (map_w, map_h) = if long_edge > SHADOW_MAP_LONG_EDGE {
- let scale = SHADOW_MAP_LONG_EDGE / long_edge;
- size_trunc(width as f64 * scale, height as f64 * scale)
- } else {
- (width, height)
- };
- let small = cv::resize_u8(channel, map_w, map_h, cv::Interp::Area)?;
-
- let k = ((map_w.max(map_h) as f64 * SHADOW_KERNEL_FRAC) as i32).max(3) | 1;
- let kernel = cv::ellipse_kernel(k)?;
- let closed = cv::morphology_close(&small, &kernel)?;
-
- let closed_f = closed.to_f32();
- let map_small = cv::box_filter_f32(&cv::box_filter_f32(&closed_f, k, k)?, k, k)?;
-
- let paper = cv::percentile_f32(&map_small, SHADOW_PAPER_PERCENTILE)?;
- if paper < 1.0 {
- return Ok(channel.clone());
- }
- let map = cv::resize_f32(&map_small, width, height, cv::Interp::Bilinear)?;
-
- let knee = paper / SHADOW_MAX_GAIN;
- let mut out = ImageU8::zeros(width, height, 1)?;
- cv::pointwise3(
- &mut out.data,
- 1,
- &channel.data,
- 1,
- &map.data,
- 1,
- |out, src, map| {
- for ((o, &v), &m) in out.iter_mut().zip(src).zip(map) {
- let m = m.max(1.0);
- let gain = if m >= knee {
- paper / m
- } else {
- 1.0 + (SHADOW_MAX_GAIN - 1.0) * (m / knee)
- };
- *o = cv::saturate_u8_f32(v as f32 * gain);
- }
- },
- );
- Ok(out)
-}
-
-fn multi_scale_retinex_on_l(bgr: &ImageU8) -> OpResult<ImageU8> {
- let lab = cv::bgr_to_lab(bgr)?;
- let mut lab_channels = cv::split_u8(&lab)?;
- let l = remove_shadows(&lab_channels[0])?;
-
- let l_original_float = l.to_f32();
- let l_float = l_original_float.map(|v| v + 1.0);
-
- let scale_factor = 2.0;
- let small_w = l_float.width as f64 / scale_factor;
- let small_h = l_float.height as f64 / scale_factor;
- let (small_cols, small_rows) = size_trunc(small_w, small_h);
-
- let l_small = cv::resize_f32(&l_float, small_cols, small_rows, cv::Interp::Area)?;
- let log_l_small = l_small.map(f32::ln);
-
- let max_dim_small = small_w.max(small_h);
- let kernel_sizes = [
- max_dim_small / 80.0,
- max_dim_small / 10.0,
- max_dim_small / 2.0,
- ];
-
- let weight = 1.0 / kernel_sizes.len() as f64;
- let mut retinex_small = ImageF32::zeros(l_small.width, l_small.height, 1)?;
-
- for ks in kernel_sizes {
- let k = (ks as i32).max(3) | 1;
- let mut term = cv::box_filter_f32(&log_l_small, k, k)?;
- term.zip_mut(&log_l_small, |blurred, log_l| *blurred = log_l - *blurred)?;
- retinex_small.zip_mut(&term, |acc, detail| {
- *acc = (*acc as f64 + detail as f64 * weight) as f32
- })?;
- }
-
- let (min_val, max_val) = cv::min_max_loc_f32(&retinex_small)?;
- let range = max_val - min_val;
- let inv_range = if range > 1e-6 { 1.0 / range } else { 1.0 };
- retinex_small.map_mut(|v| ((v - min_val as f32) as f64 * inv_range) as f32);
-
- let retinex_norm = cv::resize_f32(
- &retinex_small,
- l_float.width,
- l_float.height,
- cv::Interp::Bicubic,
- )?;
-
- let mean_l = cv::mean_f32(&l_original_float)?;
- let amplitude = 60.0;
- let alpha = 0.6;
- let bias = (mean_l - amplitude / 2.0) as f32;
-
- let corrected_l = l_original_float.zip_map(&retinex_norm, |original, detail| {
- let corrected = (detail as f64 * amplitude) as f32 + bias;
- (original as f64 * (1.0 - alpha) + corrected as f64 * alpha) as f32
- })?;
-
- let p_low_orig = percentile_l(&l_original_float, 0.001)?;
- let p_low = percentile_l(&corrected_l, 0.001)?;
- let p_high = percentile_l(&corrected_l, 0.995)?;
-
- let target_low = p_low.min(p_low_orig);
- let target_high = 245.0;
- let scale = (target_high - target_low) / (p_high - p_low + 1e-6);
-
- lab_channels[0] = corrected_l
- .map_to_u8(|v| clamp_255(((v - p_low as f32) as f64 * scale) as f32 + target_low as f32));
-
- let merged = cv::merge_u8(&lab_channels)?;
- cv::lab_to_bgr(&merged)
-}
-
-pub(crate) fn percentile_l(l: &ImageF32, p: f64) -> OpResult<f64> {
- let hist = cv::hist_256_f32(l)?;
- let total = l.pixels() as f64;
- let mut sum = 0.0;
- for (i, count) in hist.iter().enumerate() {
- sum += count;
- if sum / total >= p {
- return Ok(i as f64);
- }
- }
- Ok(255.0)
-}
-
-fn enhance_grayscale_image(img: &ImageU8) -> OpResult<ImageU8> {
- let gray = match img.channels {
- 3 => cv::bgr_to_gray(img)?,
- 1 => img.clone(),
- other => {
- return Err(format!(
- "grayscale enhancement expects a 1- or 3-channel image, got {other}"
- ));
- }
- };
- let gray = remove_shadows(&gray)?;
-
- let max_dim = gray.width.max(gray.height) as f64;
-
- let gray_f = gray.to_f32();
- let img_float = gray_f.map(|v| v + 1.0);
- let log_img = img_float.map(f32::ln);
-
- let kernel_sizes = [max_dim / 6.0, max_dim / 50.0];
- let weight = 1.0 / kernel_sizes.len() as f64;
- let mut retinex = ImageF32::zeros(gray.width, gray.height, 1)?;
-
- for kernel_size in kernel_sizes {
- let k = (kernel_size as i32).max(3) | 1;
- let mut term = cv::box_filter_f32(&img_float, k, k)?;
- term.map_mut(|v| (v + 1.0).ln());
- term.zip_mut(&log_img, |log_blur, log_v| *log_blur = log_v - *log_blur)?;
- retinex.zip_mut(&term, |acc, detail| {
- *acc = (*acc as f64 + detail as f64 * weight) as f32
- })?;
- }
-
- retinex.map_mut(f32::exp);
-
- let (p_low, p_high) = cv::percentile_pair_f32(&retinex, 0.004, 0.99)?;
- let (p_low, p_high) = (p_low as f64, p_high as f64);
- let scale = if p_high > p_low {
- 255.0 / (p_high - p_low)
- } else {
- 1.0
- };
- let result8u = retinex.map_to_u8(|v| clamp_255(((v - p_low as f32) as f64 * scale) as f32));
-
- let hist = cv::hist_256_u8(&result8u)?;
- let mut mode_val = 220usize;
- let mut mode_count = 0.0f64;
- for (i, &c) in hist.iter().enumerate().take(256).skip(180) {
- if c > mode_count {
- mode_count = c;
- mode_val = i;
- }
- }
-
- let stretched8u = if mode_val >= 254 {
- let (g_low, g_high) = cv::percentile_pair_f32(&gray_f, 0.01, 0.99)?;
- let (g_low, g_high) = (g_low as f64, g_high as f64);
- let scale = 255.0 / (g_high - g_low + 1e-6);
- gray_f.map_to_u8(|v| clamp_255(((v - g_low as f32) as f64 * scale) as f32))
- } else {
- let scale = 255.0 / mode_val as f64;
- result8u.to_f32().map_to_u8(|v| {
- let scaled = (v as f64 * scale) as f32;
- if scaled < 255.0 { scaled } else { 255.0 }
- })
- };
-
- let denoised = cv::bilateral_filter_u8(&stretched8u, 9, 20.0, 10.0)?;
- cv::gray_to_bgr(&denoised)
-}
-
-#[cfg(test)]
-mod tests {
- use super::remove_shadows;
- use crate::cv::image::ImageU8;
-
- fn background_mean(img: &ImageU8, x0: i32, x1: i32) -> f64 {
- let mut sum = 0.0;
- let mut count = 0.0;
- for y in 20..img.height - 20 {
- for x in x0..x1 {
- if x % 17 != 3 || y % 11 != 5 {
- sum += img.data[(y * img.width + x) as usize] as f64;
- count += 1.0;
- }
- }
- }
- sum / count
- }
-
- #[test]
- fn remove_shadows_lifts_shadowed_paper_and_keeps_text_dark() {
- let (w, h) = (400, 300);
- let mut data = vec![230u8; (w * h) as usize];
- for y in 0..h {
- for x in 0..w {
- let i = (y * w + x) as usize;
- if x % 17 == 3 && y % 11 == 5 {
- data[i] = 40;
- }
- if x < w / 2 {
- data[i] = (data[i] as f64 * 0.55) as u8;
- }
- }
- }
- let img = ImageU8::new(w, h, 1, data).unwrap();
- let out = remove_shadows(&img).unwrap();
-
- let shadow_bg = background_mean(&out, 20, w / 4);
- let lit_bg = background_mean(&out, 3 * w / 4, w - 20);
- assert!(
- (shadow_bg - lit_bg).abs() < 12.0,
- "shadowed paper {shadow_bg:.1} should match lit paper {lit_bg:.1}"
- );
-
- let mut min_shadow_half = 255u8;
- for y in 20..h - 20 {
- for x in 20..w / 4 {
- min_shadow_half = min_shadow_half.min(out.data[(y * w + x) as usize]);
- }
- }
- assert!(
- min_shadow_half < 90,
- "text in the lifted shadow should stay dark, got {min_shadow_half}"
- );
- }
-
- #[test]
- fn remove_shadows_is_near_identity_on_evenly_lit_paper() {
- let (w, h) = (400, 300);
- let mut data = vec![225u8; (w * h) as usize];
- for y in 0..h {
- for x in 0..w {
- if x % 17 == 3 && y % 11 == 5 {
- data[(y * w + x) as usize] = 40;
- }
- }
- }
- let img = ImageU8::new(w, h, 1, data.clone()).unwrap();
- let out = remove_shadows(&img).unwrap();
- let mean_delta: f64 = out
- .data
- .iter()
- .zip(&data)
- .map(|(&a, &b)| (a as f64 - b as f64).abs())
- .sum::<f64>()
- / data.len() as f64;
- assert!(mean_delta < 4.0, "expected near-identity, got {mean_delta}");
- }
-}
diff --git a/rust/crates/ml/src/scan/quad_score.rs b/rust/crates/ml/src/scan/quad_score.rs
deleted file mode 100644
index 7a766cfc82..0000000000
--- a/rust/crates/ml/src/scan/quad_score.rs
+++ /dev/null
@@ -1,33 +0,0 @@
-use super::OpResult;
-use super::geometry::Point;
-use crate::cv;
-use crate::cv::image::{ImageF32, ImageU8};
-
-pub(crate) fn make_polygon_mask(width: i32, height: i32, polygon: &[Point]) -> OpResult<ImageU8> {
- let pts: Vec<(i32, i32)> = polygon.iter().map(|p| (p.x as i32, p.y as i32)).collect();
- cv::fill_poly(width, height, &pts, 1.0)
-}
-
-pub(crate) fn score_quad_against_probmap(
- quad: &[Point],
- probmap: &ImageF32,
- min_quad_area_ratio: f64,
-) -> OpResult<f64> {
- let mask = make_polygon_mask(probmap.width, probmap.height, quad)?;
-
- let mut sum_masked = 0.0f64;
- let mut sum_mask = 0.0f64;
- for (&prob, &inside) in probmap.data.iter().zip(mask.data.iter()) {
- let inside = inside as f32;
- sum_masked += (prob * inside) as f64;
- sum_mask += inside as f64;
- }
- let mean_prob = sum_masked / sum_mask;
- let area_ratio = sum_mask / (probmap.height * probmap.width) as f64;
-
- Ok(if area_ratio < min_quad_area_ratio {
- 0.0
- } else {
- mean_prob * (0.7 + 0.3 * area_ratio)
- })
-}
diff --git a/rust/crates/ml/src/scan/scanner.rs b/rust/crates/ml/src/scan/scanner.rs
index 258c6d33c3..600ad79932 100644
--- a/rust/crates/ml/src/scan/scanner.rs
+++ b/rust/crates/ml/src/scan/scanner.rs
@@ -1,12 +1,12 @@
use thiserror::Error;
+use super::appearance::render_document;
+use super::boundary::{SearchBudget, locate, refine_capture};
use super::codec;
-use super::color::{ColorMode, auto_color_mode, quad_to_image};
-use super::detection::{detect_document_quad, extract_document, resize_for_max_pixels};
-use super::geometry::{ImageSize, Quad};
-use super::mask::Mask;
-use super::segmentation::{MASK_SIDE, Segmenter};
+use super::page::{RenderPlan, SourceExtent, quarter_turns, rotate_mask_quad};
+use super::segmentation::{MASK_SIDE, ProbabilityMap, Segmenter};
use super::yuv::{PlaneLayout, bgra_to_bgr, yuv420_to_bgr};
+use super::{ColorMode, Quad};
use crate::cv::image::ImageU8;
const DEFAULT_MAX_PIXELS: u32 = 2_000_000;
@@ -73,12 +73,10 @@ impl ScannerSession {
})
}
- fn segment(&self, bgr: &ImageU8) -> Result<Mask, ScanError> {
- let probmap = self
- .segmenter
- .probability_map_u8(bgr)
- .map_err(ScanError::Pipeline)?;
- Ok(Mask::from_probmap(probmap, MASK_SIDE, MASK_SIDE))
+ fn segment(&self, bgr: &ImageU8) -> Result<ProbabilityMap, ScanError> {
+ self.segmenter
+ .probability_map(bgr)
+ .map_err(ScanError::Pipeline)
}
pub fn live_detect_bgra(
@@ -91,7 +89,7 @@ impl ScannerSession {
) -> Result<Option<Quad>, ScanError> {
let bgr = bgra_to_bgr(bgra, to_i32(row_stride)?, to_i32(width)?, to_i32(height)?)
.map_err(ScanError::InvalidInput)?;
- let frame = ImageSize::new(bgr.width as f64, bgr.height as f64);
+ let frame = SourceExtent::new(bgr.width, bgr.height).map_err(ScanError::InvalidInput)?;
self.live_detect(&bgr, frame, rotation_degrees)
}
@@ -105,21 +103,28 @@ impl ScannerSession {
) -> Result<Option<Quad>, ScanError> {
let bgr = yuv420_to_bgr(y, u, v, layout, MASK_SIDE, MASK_SIDE)
.map_err(ScanError::InvalidInput)?;
- let frame = ImageSize::new(layout.width as f64, layout.height as f64);
+ let frame =
+ SourceExtent::new(layout.width, layout.height).map_err(ScanError::InvalidInput)?;
self.live_detect(&bgr, frame, rotation_degrees)
}
fn live_detect(
&self,
bgr: &ImageU8,
- frame_size: ImageSize,
+ frame_size: SourceExtent,
rotation_degrees: i32,
) -> Result<Option<Quad>, ScanError> {
+ let turns = quarter_turns(rotation_degrees).map_err(ScanError::InvalidInput)?;
let mask = self.segment(bgr)?;
- let quad_in_mask =
- detect_document_quad(&mask, frame_size, true).map_err(ScanError::Pipeline)?;
- let mask_size = ImageSize::new(mask.width as f64, mask.height as f64);
- Ok(quad_in_mask.map(|q| q.rotate90(rotation_degrees / 90, mask_size)))
+ let detection = locate(&mask, frame_size, SearchBudget::Live);
+ log::debug!(
+ "live detection: {}, evidence {:.3}",
+ detection.reason,
+ detection.confidence
+ );
+ Ok(detection
+ .quad
+ .map(|quad| rotate_mask_quad(quad.corners, turns, MASK_SIDE as f64)))
}
pub fn process_capture(
@@ -150,25 +155,44 @@ impl ScannerSession {
image_bytes: &[u8],
max_pixels: Option<u32>,
) -> Result<ScanResult, ScanError> {
+ let budget = max_pixels.unwrap_or(DEFAULT_MAX_PIXELS);
+ validate_budget(budget)?;
let bgr = codec::decode_bgr(image_bytes)?;
- let max_pixels = max_pixels.unwrap_or(DEFAULT_MAX_PIXELS) as f64;
-
+ let extent = SourceExtent::new(bgr.width, bgr.height).map_err(ScanError::InvalidInput)?;
let mask = self.segment(&bgr)?;
- let original_size = ImageSize::new(bgr.width as f64, bgr.height as f64);
- let quad_in_mask =
- detect_document_quad(&mask, original_size, false).map_err(ScanError::Pipeline)?;
-
- let Some(quad_in_mask) = quad_in_mask else {
- let page = resize_for_max_pixels(&bgr, max_pixels).map_err(ScanError::Pipeline)?;
- return finish(None, ColorMode::Color, &bgr, &page, DEFAULT_JPEG_QUALITY);
+ let detection = locate(&mask, extent, SearchBudget::Capture);
+ log::debug!(
+ "capture detection: {}, evidence {:.3}",
+ detection.reason,
+ detection.confidence
+ );
+ let quad = detection.quad.map(|quad| quad.in_source(extent));
+ let quad = match quad {
+ Some(quad) => refine_capture(&bgr, quad).map_err(ScanError::Pipeline)?,
+ None => None,
};
-
- let quad = quad_to_image(&quad_in_mask, &mask, bgr.size());
- let color_mode = auto_color_mode(&bgr, &mask, &quad).map_err(ScanError::Pipeline)?;
- let page = extract_document(&bgr, &quad, 0, color_mode, max_pixels)
+ let Some(quad) = quad else {
+ let plan = RenderPlan::new(extent.full_frame(), extent, 0, budget)
+ .map_err(ScanError::Pipeline)?;
+ if plan.width == bgr.width && plan.height == bgr.height {
+ return finish(None, ColorMode::Color, extent, bgr, DEFAULT_JPEG_QUALITY);
+ }
+ let page = plan.render(&bgr).map_err(ScanError::Pipeline)?;
+ drop(bgr);
+ return finish(
+ None,
+ ColorMode::Color,
+ extent,
+ page.image,
+ DEFAULT_JPEG_QUALITY,
+ );
+ };
+ let plan = RenderPlan::new(quad, extent, 0, budget).map_err(ScanError::Pipeline)?;
+ let mut page = plan.render(&bgr).map_err(ScanError::Pipeline)?;
+ drop(bgr);
+ let mode = render_document(&mut page.image, page.valid.as_deref(), None)
.map_err(ScanError::Pipeline)?;
-
- finish(Some(quad), color_mode, &bgr, &page, DEFAULT_JPEG_QUALITY)
+ finish(Some(quad), mode, extent, page.image, DEFAULT_JPEG_QUALITY)
}
pub fn reprocess(
@@ -195,26 +219,40 @@ impl ScannerSession {
source_bytes: &[u8],
options: &ReprocessOptions,
) -> Result<ScanResult, ScanError> {
+ let budget = options.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS);
+ validate_budget(budget)?;
+ quarter_turns(options.rotation_degrees).map_err(ScanError::InvalidInput)?;
let bgr = codec::decode_bgr(source_bytes)?;
- let page = extract_document(
- &bgr,
- &options.quad,
- options.rotation_degrees,
- options.color_mode,
- options.max_pixels.unwrap_or(DEFAULT_MAX_PIXELS) as f64,
+ let extent = SourceExtent::new(bgr.width, bgr.height).map_err(ScanError::InvalidInput)?;
+ let plan = RenderPlan::new(options.quad, extent, options.rotation_degrees, budget)
+ .map_err(ScanError::InvalidInput)?;
+ let mut page = plan.render(&bgr).map_err(ScanError::Pipeline)?;
+ drop(bgr);
+ let mode = render_document(
+ &mut page.image,
+ page.valid.as_deref(),
+ Some(options.color_mode),
)
.map_err(ScanError::Pipeline)?;
-
finish(
Some(options.quad),
- options.color_mode,
- &bgr,
- &page,
+ mode,
+ extent,
+ page.image,
DEFAULT_JPEG_QUALITY,
)
}
}
+fn validate_budget(budget: u32) -> Result<(), ScanError> {
+ if budget == 0 {
+ return Err(ScanError::InvalidInput(
+ "pixel budget must be positive".to_owned(),
+ ));
+ }
+ Ok(())
+}
+
fn basename(path: &str) -> &str {
std::path::Path::new(path)
.file_name()
@@ -230,19 +268,76 @@ fn to_i32(value: u32) -> Result<i32, ScanError> {
fn finish(
quad: Option<Quad>,
color_mode: ColorMode,
- source: &ImageU8,
- page: &ImageU8,
+ source: SourceExtent,
+ page: ImageU8,
jpeg_quality: u8,
) -> Result<ScanResult, ScanError> {
+ let output_width = page.width as u32;
+ let output_height = page.height as u32;
let processed_image = codec::encode_jpeg(page, jpeg_quality)?;
Ok(ScanResult {
quad,
color_mode,
- output_width: page.width as u32,
- output_height: page.height as u32,
+ output_width,
+ output_height,
source_width: source.width as u32,
source_height: source.height as u32,
processed_image,
})
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn reprocess_needs_no_model_and_repeated_original_rotations_do_not_accumulate_loss()
+ -> Result<(), ScanError> {
+ let session = ScannerSession {
+ segmenter: Segmenter::unavailable_for_render_tests(),
+ };
+ let source = ImageU8::new(
+ 120,
+ 80,
+ 3,
+ (0..120 * 80 * 3)
+ .map(|i| ((i * 37 + i / 17 * 31) % 256) as u8)
+ .collect(),
+ )
+ .map_err(ScanError::Codec)?;
+ let bytes = codec::encode_jpeg(source, 90)?;
+ let extent = SourceExtent::new(120, 80).map_err(ScanError::InvalidInput)?;
+ let mut options = ReprocessOptions {
+ quad: extent.full_frame(),
+ rotation_degrees: 0,
+ color_mode: ColorMode::Color,
+ max_pixels: Some(6000),
+ };
+ let first = session.reprocess(&bytes, &options)?;
+ assert_eq!(first, session.reprocess(&bytes, &options)?);
+ for rotation in [90, 180, 270, 360] {
+ options.rotation_degrees = rotation;
+ let result = session.reprocess(&bytes, &options)?;
+ assert_eq!((result.source_width, result.source_height), (120, 80));
+ assert_eq!(result.quad, Some(options.quad));
+ assert_eq!(result.color_mode, ColorMode::Color);
+ assert!(result.output_width * result.output_height <= 6000);
+ if rotation % 180 == 90 {
+ assert_eq!(
+ (result.output_width, result.output_height),
+ (first.output_height, first.output_width)
+ );
+ }
+ if rotation == 360 {
+ assert_eq!(result.processed_image, first.processed_image);
+ }
+ }
+ options.max_pixels = Some(0);
+ assert!(matches!(
+ session.reprocess(&bytes, &options),
+ Err(ScanError::InvalidInput(_))
+ ));
+ Ok(())
+ }
+}
diff --git a/rust/crates/ml/src/scan/segmentation.rs b/rust/crates/ml/src/scan/segmentation.rs
index 787d16fad9..ef968ac449 100644
--- a/rust/crates/ml/src/scan/segmentation.rs
+++ b/rust/crates/ml/src/scan/segmentation.rs
@@ -52,32 +52,193 @@ impl Segmenter {
Ok(data)
}
- pub(crate) fn probability_map_u8(&self, bgr: &ImageU8) -> OpResult<Vec<u8>> {
- if bgr.channels != 3 {
- return Err(format!(
- "probability_map_u8: expected a 3-channel image, got {}",
- bgr.channels
- ));
+ pub(crate) fn probability_map(&self, bgr: &ImageU8) -> OpResult<ProbabilityMap> {
+ ProbabilityMap::new(
+ self.infer(prepare_input(bgr)?)?,
+ MASK_SIDE as usize,
+ MASK_SIDE as usize,
+ )
+ }
+}
+
+pub(super) fn prepare_input(bgr: &ImageU8) -> OpResult<Vec<f32>> {
+ if bgr.channels != 3 {
+ return Err(format!(
+ "segmentation requires three channels, got {}",
+ bgr.channels
+ ));
+ }
+ let resized = if bgr.width == MASK_SIDE && bgr.height == MASK_SIDE {
+ None
+ } else {
+ Some(cv::resize_u8(
+ bgr,
+ MASK_SIDE,
+ MASK_SIDE,
+ cv::Interp::Bilinear,
+ )?)
+ };
+ let resized = resized.as_ref().unwrap_or(bgr);
+ let mut input = vec![0.0f32; resized.data.len()];
+ for (out, px) in input
+ .as_chunks_mut::<3>()
+ .0
+ .iter_mut()
+ .zip(resized.data.as_chunks::<3>().0.iter())
+ {
+ out[0] = (px[2] as f32 - 127.5) / 127.5;
+ out[1] = (px[1] as f32 - 127.5) / 127.5;
+ out[2] = (px[0] as f32 - 127.5) / 127.5;
+ }
+ Ok(input)
+}
+
+pub(super) struct ProbabilityMap {
+ pub width: usize,
+ pub height: usize,
+ pub values: Vec<f32>,
+}
+
+impl ProbabilityMap {
+ pub fn new(mut values: Vec<f32>, width: usize, height: usize) -> OpResult<Self> {
+ if width == 0 || height == 0 || width.checked_mul(height) != Some(values.len()) {
+ return Err("invalid probability-map dimensions".to_owned());
}
- let resized = cv::resize_u8(bgr, MASK_SIDE, MASK_SIDE, cv::Interp::Bilinear)?;
-
- let mut input = vec![0.0f32; resized.data.len()];
- for (out, px) in input
- .as_chunks_mut::<3>()
- .0
- .iter_mut()
- .zip(resized.data.as_chunks::<3>().0.iter())
- {
- out[0] = (px[2] as f32 - 127.5) / 127.5;
- out[1] = (px[1] as f32 - 127.5) / 127.5;
- out[2] = (px[0] as f32 - 127.5) / 127.5;
+ if values.iter().any(|v| !v.is_finite()) {
+ return Err("non-finite segmentation output".to_owned());
}
+ values.iter_mut().for_each(|v| *v = v.clamp(0.0, 1.0));
+ Ok(Self {
+ width,
+ height,
+ values,
+ })
+ }
+}
- let data = self.infer(input)?;
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::scan::yuv::{PlaneLayout, bgra_to_bgr, yuv420_to_bgr};
- Ok(data
+ fn hash(image: &ImageU8) -> OpResult<u64> {
+ Ok(prepare_input(image)?
.iter()
- .map(|&v| (v.clamp(0.0, 1.0) * 255.0).round_ties_even() as u8)
- .collect())
+ .flat_map(|v| v.to_le_bytes())
+ .fold(0xcbf29ce484222325u64, |h, b| {
+ (h ^ b as u64).wrapping_mul(0x100000001b3)
+ }))
+ }
+
+ #[test]
+ fn prepared_input_matches_pre_rewrite_golden_bits() -> OpResult<()> {
+ for (w, h, expected) in [
+ (1, 1, 0x194e6f7e1cf62325),
+ (17, 39, 0x66f02b798fcce024),
+ (301, 103, 0x70448e7edd3ec825),
+ (256, 256, 0x6e6f7ff897d8c759),
+ (1027, 769, 0x3872cf1d817be804),
+ ] {
+ let bytes = (0..w * h * 3)
+ .map(|i| ((i * 71 + i / 17 * 19) % 256) as u8)
+ .collect();
+ assert_eq!(hash(&ImageU8::new(w, h, 3, bytes)?)?, expected);
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn camera_input_matches_pre_rewrite_golden_bits() -> OpResult<()> {
+ let (w, h, stride) = (37, 21, 37 * 4 + 13);
+ let bgra = (0..stride * h)
+ .map(|i| ((i * 97 + i / 11 * 37) % 256) as u8)
+ .collect::<Vec<_>>();
+ assert_eq!(
+ hash(&bgra_to_bgr(&bgra, stride, w, h)?)?,
+ 0xa01f90e4cc066c9c
+ );
+ for (step, expected) in [(1, 0xbd51f44eacc387b7), (2, 0x8078d82c5cf79ac1)] {
+ let ys = 42;
+ let uvs = 19 * step + 7;
+ let y = (0..ys * h)
+ .map(|i| ((i * 37 + i / 9) % 256) as u8)
+ .collect::<Vec<_>>();
+ let u = (0..uvs * 11)
+ .map(|i| ((i * 13 + 75) % 256) as u8)
+ .collect::<Vec<_>>();
+ let v = (0..uvs * 11)
+ .map(|i| ((i * 23 + 101) % 256) as u8)
+ .collect::<Vec<_>>();
+ let layout = PlaneLayout {
+ width: w,
+ height: h,
+ y_row_stride: ys,
+ uv_row_stride: uvs,
+ uv_pixel_stride: step,
+ };
+ assert_eq!(
+ hash(&yuv420_to_bgr(&y, &u, &v, layout, 256, 256)?)?,
+ expected
+ );
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn probabilities_reject_non_finite_and_bad_shape() {
+ assert!(ProbabilityMap::new(vec![f32::NAN], 1, 1).is_err());
+ assert!(ProbabilityMap::new(vec![f32::INFINITY], 1, 1).is_err());
+ assert!(ProbabilityMap::new(vec![0.0], 2, 2).is_err());
+ assert!(ProbabilityMap::new(vec![], 0, 0).is_err());
+ }
+}
+
+#[cfg(test)]
+mod model_parity_tests {
+ use super::*;
+
+ #[test]
+ #[ignore = "requires ENTE_SCAN_MODEL and ENTE_SCAN_INPUT_GOLDENS"]
+ fn golden_tensor_bytes_and_same_provider_outputs_match() -> OpResult<()> {
+ let model = std::env::var("ENTE_SCAN_MODEL").map_err(|e| e.to_string())?;
+ let directory = std::env::var("ENTE_SCAN_INPUT_GOLDENS").map_err(|e| e.to_string())?;
+ let segmenter = Segmenter::new(&model).map_err(|e| e.to_string())?;
+ for (w, h) in [(1, 1), (17, 39), (301, 103), (256, 256), (1027, 769)] {
+ let data = (0..w * h * 3)
+ .map(|i| ((i * 71 + i / 17 * 19) % 256) as u8)
+ .collect();
+ let prepared = prepare_input(&ImageU8::new(w, h, 3, data)?)?;
+ let golden_bytes =
+ std::fs::read(format!("{directory}/bgr-{w}-{h}.f32")).map_err(|e| e.to_string())?;
+ let actual_bytes: Vec<u8> = prepared.iter().flat_map(|v| v.to_le_bytes()).collect();
+ assert_eq!(actual_bytes, golden_bytes);
+ let golden = golden_bytes
+ .as_chunks::<4>()
+ .0
+ .iter()
+ .map(|&b| f32::from_le_bytes(b))
+ .collect();
+ let expected = segmenter.infer(golden)?;
+ let actual = segmenter.infer(prepared)?;
+ assert_eq!(actual, expected);
+ println!(
+ "same-provider input/output parity {w}x{h}: {} float samples",
+ actual.len()
+ );
+ }
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+impl Segmenter {
+ pub(super) fn unavailable_for_render_tests() -> Self {
+ Self {
+ session: Mutex::new(OnnxSession::new(
+ "/nonexistent/document-segmentation.onnx",
+ MODEL_NAMESPACE,
+ ExecutionMode::PlatformDefault,
+ )),
+ }
}
}
diff --git a/rust/crates/ml/src/scan/yuv.rs b/rust/crates/ml/src/scan/yuv.rs
index 745f11a114..feda3e7cd7 100644
--- a/rust/crates/ml/src/scan/yuv.rs
+++ b/rust/crates/ml/src/scan/yuv.rs
@@ -18,7 +18,7 @@ const BG: i32 = UG * 128 + VG * 128 + YGB;
const BR: i32 = VR * 128 + YGB;
fn ceil_half(value: i32) -> i32 {
- (value + 1) / 2
+ value / 2 + value % 2
}
fn clamp_u8(value: i32) -> u8 {
@@ -44,11 +44,14 @@ pub struct PlaneLayout {
pub uv_pixel_stride: i32,
}
-fn required_len(rows: i32, row_stride: i32, last_row_bytes: i32) -> usize {
- if rows <= 0 {
- return 0;
+fn required_len(rows: i32, row_stride: i32, last_row_bytes: i32) -> Result<usize, String> {
+ if rows <= 0 || row_stride < 0 || last_row_bytes < 0 {
+ return Err("invalid plane dimensions".to_owned());
}
- (rows - 1) as usize * row_stride as usize + last_row_bytes as usize
+ ((rows - 1) as usize)
+ .checked_mul(row_stride as usize)
+ .and_then(|length| length.checked_add(last_row_bytes as usize))
+ .ok_or_else(|| "plane size overflow".to_owned())
}
fn check_layout(
@@ -78,15 +81,19 @@ fn check_layout(
}
let cw = ceil_half(width);
let ch = ceil_half(height);
- if uv_row_stride < (cw - 1) * uv_pixel_stride + 1 {
+ let last_chroma_byte = (cw - 1)
+ .checked_mul(uv_pixel_stride)
+ .and_then(|value| value.checked_add(1))
+ .ok_or_else(|| "chroma stride overflow".to_owned())?;
+ if uv_row_stride < last_chroma_byte {
return Err(format!(
"uv row stride {uv_row_stride} is too small for {cw} chroma samples \
at pixel stride {uv_pixel_stride}"
));
}
- let need_y = required_len(height, y_row_stride, width);
- let need_uv = required_len(ch, uv_row_stride, (cw - 1) * uv_pixel_stride + 1);
+ let need_y = required_len(height, y_row_stride, width)?;
+ let need_uv = required_len(ch, uv_row_stride, last_chroma_byte)?;
if y_plane.len() < need_y {
return Err(format!(
"y plane has {} bytes, needs {need_y}",
@@ -192,7 +199,7 @@ pub(crate) fn bgra_to_bgr(
));
}
let row_stride = row_stride as usize;
- let needed = required_len(height, row_stride as i32, row_bytes as i32);
+ let needed = required_len(height, row_stride as i32, row_bytes as i32)?;
if bgra.len() < needed {
return Err(format!(
"BGRA buffer has {} bytes, needs {needed}",
@@ -551,3 +558,25 @@ mod tests {
assert!(bgra_to_bgr(&padded, stride as i32, 0, 2).is_err());
}
}
+
+#[cfg(test)]
+mod invalid_metadata_tests {
+ use super::*;
+
+ #[test]
+ fn huge_camera_metadata_is_rejected_without_overflow() {
+ for width in [1, 3, i32::MAX] {
+ for stride in [1, i32::MAX] {
+ let layout = PlaneLayout {
+ width,
+ height: i32::MAX,
+ y_row_stride: i32::MAX,
+ uv_row_stride: stride,
+ uv_pixel_stride: i32::MAX,
+ };
+ assert!(yuv420_to_bgr(&[], &[], &[], layout, 256, 256).is_err());
+ assert!(bgra_to_bgr(&[], stride, width, i32::MAX).is_err());
+ }
+ }
+ }
+}