Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[rust] Detect documents against the probability map, not a binarized mask

laurenspriemee9ecfe59e92
diff --git a/rust/crates/ml/src/scan/color.rs b/rust/crates/ml/src/scan/color.rs
index d7599c04c8..ed16fd58b4 100644
--- a/rust/crates/ml/src/scan/color.rs
+++ b/rust/crates/ml/src/scan/color.rs
@@ -83,7 +83,7 @@ fn document_mask(
orig_size: (i32, i32),
work_size: (i32, i32),
) -> OpResult<ImageU8> {
- let mask_image = mask.to_image()?;
+ let mask_image = mask.binary_image()?;
let resized_mask =
cv::resize_area(ImageRef::U8(&mask_image), work_size.0, work_size.1)?.into_u8()?;
diff --git a/rust/crates/ml/src/scan/detection.rs b/rust/crates/ml/src/scan/detection.rs
index bd92f5f429..e1ea0e16a7 100644
--- a/rust/crates/ml/src/scan/detection.rs
+++ b/rust/crates/ml/src/scan/detection.rs
@@ -21,7 +21,7 @@ pub(crate) fn detect_document_quad(
original_size: ImageSize,
is_live_analysis: bool,
) -> OpResult<Option<Quad>> {
- let mask_image = mask.to_image()?;
+ let prob_image = mask.prob_image()?;
let thresholds: &[f64] = if is_live_analysis {
&LIVE_THRESHOLDS
} else {
@@ -29,11 +29,11 @@ pub(crate) fn detect_document_quad(
};
let mut vertices =
- find_quad_from_orientation_with_adaptive_threshold(&mask_image, original_size, thresholds)?;
+ find_quad_from_orientation_with_adaptive_threshold(&prob_image, original_size, thresholds)?;
if vertices.is_none() && !is_live_analysis {
// Fallback: min-area rectangle of the biggest contour.
- if let Some(biggest) = biggest_contour(&mask_image)? {
+ if let Some(biggest) = biggest_contour(&mask.binary_image()?)? {
vertices = min_area_rect(&biggest.points, mask.width, mask.height);
}
}
@@ -45,14 +45,14 @@ pub(crate) fn detect_document_quad(
}
fn find_quad_from_orientation_with_adaptive_threshold(
- mask_image: &ImageU8,
+ prob_image: &ImageU8,
original_size: ImageSize,
thresholds: &[f64],
) -> OpResult<Option<Vec<Point>>> {
- let probmap_smooth = cv::gaussian_blur_u8(mask_image, 3)?;
+ let probmap_smooth = cv::gaussian_blur_u8(prob_image, 3)?;
let kernel = cv::ellipse_kernel(5)?;
- let prob_float = cv::u8_to_f32(mask_image)?;
+ let prob_float = cv::u8_to_f32(prob_image)?;
let mut best_quad: Option<Vec<Point>> = None;
let mut best_score = 0.0f64;
diff --git a/rust/crates/ml/src/scan/mask.rs b/rust/crates/ml/src/scan/mask.rs
index 286219878e..e43e08e687 100644
--- a/rust/crates/ml/src/scan/mask.rs
+++ b/rust/crates/ml/src/scan/mask.rs
@@ -1,26 +1,30 @@
use super::OpResult;
+use crate::cv;
use crate::cv::image::ImageU8;
+/// The segmentation probabilities, kept at full precision: the detection
+/// threshold sweep and the quad scoring both need the model's confidence, not
+/// just its argmax.
pub(crate) struct Mask {
pub width: i32,
pub height: i32,
- binary: Vec<u8>,
+ prob: Vec<u8>,
}
impl Mask {
- pub(crate) fn from_probmap(probmap: &[u8], width: i32, height: i32) -> Self {
- let binary = probmap
- .iter()
- .map(|&v| if v >= 128 { 255u8 } else { 0u8 })
- .collect();
+ pub(crate) fn from_probmap(prob: Vec<u8>, width: i32, height: i32) -> Self {
Self {
width,
height,
- binary,
+ prob,
}
}
- pub(crate) fn to_image(&self) -> OpResult<ImageU8> {
- ImageU8::new(self.width, self.height, 1, self.binary.clone())
+ pub(crate) fn prob_image(&self) -> OpResult<ImageU8> {
+ ImageU8::new(self.width, self.height, 1, self.prob.clone())
+ }
+
+ pub(crate) fn binary_image(&self) -> OpResult<ImageU8> {
+ cv::threshold_binary_u8(&self.prob_image()?, 127.0, 255.0)
}
}
diff --git a/rust/crates/ml/src/scan/scanner.rs b/rust/crates/ml/src/scan/scanner.rs
index 10f4308d0d..0854a54204 100644
--- a/rust/crates/ml/src/scan/scanner.rs
+++ b/rust/crates/ml/src/scan/scanner.rs
@@ -87,7 +87,7 @@ impl ScannerSession {
.segmenter
.probability_map_u8(bgr)
.map_err(ScanError::Pipeline)?;
- Ok(Mask::from_probmap(&probmap, MASK_SIDE, MASK_SIDE))
+ Ok(Mask::from_probmap(probmap, MASK_SIDE, MASK_SIDE))
}
pub fn live_detect_rgba(