Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-perception): drop whitespace-only OCR text instead of failing the parse (#4183)

Francesco Bonaccife76ccff0417
diff --git a/libs/cua-driver/rust/crates/cua-perception/src/lib.rs b/libs/cua-driver/rust/crates/cua-perception/src/lib.rs
index 8b2b9ca007..03cc014699 100644
--- a/libs/cua-driver/rust/crates/cua-perception/src/lib.rs
+++ b/libs/cua-driver/rust/crates/cua-perception/src/lib.rs
@@ -581,20 +581,30 @@ fn inference_result(
}),
);
}
+ // Driver validates the whole parse and rejects it if any region is empty,
+ // outside the image, or carries whitespace-only text, so drop such
+ // regions here instead of failing every region with them.
let regions = regions
.into_iter()
+ .filter_map(|region| {
+ let x = (region.bounds.x1.floor().max(0.0) as u32).min(width);
+ let y = (region.bounds.y1.floor().max(0.0) as u32).min(height);
+ let right = region.bounds.x2.ceil().min(width as f32).max(0.0) as u32;
+ let bottom = region.bounds.y2.ceil().min(height as f32).max(0.0) as u32;
+ let blank_text = region
+ .text
+ .as_deref()
+ .is_some_and(|text| text.trim().is_empty());
+ (right > x && bottom > y && !blank_text).then_some((region, x, y, right, bottom))
+ })
.enumerate()
- .map(|(index, region)| {
+ .map(|(index, (region, x, y, right, bottom))| {
let kind = region.kind;
let class_id = region.class_id;
- let x = region.bounds.x1.floor().max(0.0) as u32;
- let y = region.bounds.y1.floor().max(0.0) as u32;
- let right = region.bounds.x2.ceil().min(width as f32) as u32;
- let bottom = region.bounds.y2.ceil().min(height as f32) as u32;
let mut value = json!({
"id": format!("{}-{}", kind, index + 1),
"kind": kind,
- "bounds": { "x": x, "y": y, "width": right.saturating_sub(x), "height": bottom.saturating_sub(y) },
+ "bounds": { "x": x, "y": y, "width": right - x, "height": bottom - y },
"confidence": region.confidence,
"interactive": false
});
@@ -811,6 +821,44 @@ mod tests {
assert_eq!(decoded["text_geometry"], "axis_aligned_bounds");
}
+ #[test]
+ fn degenerate_and_blank_regions_are_dropped_before_numbering() {
+ let region = |kind: &'static str, x1: f32, x2: f32, text: Option<&str>| InferenceRegion {
+ kind,
+ bounds: Rect {
+ x1,
+ y1: 1.0,
+ x2,
+ y2: 9.0,
+ },
+ text: text.map(str::to_owned),
+ confidence: 0.9,
+ class_id: None,
+ };
+ let result = inference_result(
+ "capture-1",
+ "abc123",
+ 100,
+ 50,
+ vec![
+ region("text", 1.0, 5.0, Some(" ")),
+ region("icon", 100.0, 120.0, None),
+ region("text", 3.0, 3.0, Some("zero width")),
+ region("text", 10.0, 30.0, Some("Submit")),
+ ],
+ json!({ "backend": "test" }),
+ (env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")),
+ );
+ assert_eq!(
+ result["regions"],
+ json!([{
+ "id": "text-1", "kind": "text",
+ "bounds": { "x": 10, "y": 1, "width": 20, "height": 8 },
+ "confidence": 0.8999999761581421, "text": "Submit", "interactive": false
+ }])
+ );
+ }
+
#[test]
fn fixture_regions_explicitly_serialize_as_non_interactive() {
let result = fixture_result("capture-1", "cua-perception", "0.1.0");
diff --git a/libs/cua-driver/rust/crates/cua-perception/src/runtime.rs b/libs/cua-driver/rust/crates/cua-perception/src/runtime.rs
index 5b40a22153..530f4bfcc7 100644
--- a/libs/cua-driver/rust/crates/cua-perception/src/runtime.rs
+++ b/libs/cua-driver/rust/crates/cua-perception/src/runtime.rs
@@ -255,9 +255,12 @@ impl OnnxBackend {
reason,
},
)?;
- if text.is_empty() {
+ // The recognizer dictionary includes a space class, so a crop can
+ // decode to whitespace only. Driver rejects whitespace-only text, and
+ // one such region would invalidate the whole parse.
+ let Some(text) = recognized_text(text) else {
return Ok(None);
- }
+ };
Ok(Some(InferenceRegion {
kind: "text",
bounds: detection.bounds,
@@ -268,6 +271,18 @@ impl OnnxBackend {
}
}
+/// Trims recognized text and drops a crop that decoded to whitespace only.
+fn recognized_text(text: String) -> Option<String> {
+ let trimmed = text.trim();
+ if trimmed.is_empty() {
+ None
+ } else if trimmed.len() == text.len() {
+ Some(text)
+ } else {
+ Some(trimmed.to_owned())
+ }
+}
+
fn parse_dictionary(source: &str, format: DictionaryFormat) -> Result<Vec<String>, InferenceError> {
let values = match format {
DictionaryFormat::PlainLines => source
@@ -529,6 +544,42 @@ fn run_tensor(
mod tests {
use super::*;
+ #[test]
+ fn recognized_text_trims_and_drops_whitespace_only_crops() {
+ assert_eq!(
+ recognized_text("Submit".to_owned()).as_deref(),
+ Some("Submit")
+ );
+ assert_eq!(
+ recognized_text(" Sign in ".to_owned()).as_deref(),
+ Some("Sign in")
+ );
+ assert_eq!(recognized_text("a b".to_owned()).as_deref(), Some("a b"));
+ for blank in ["", " ", " ", "\t", " \u{3000} "] {
+ assert_eq!(recognized_text(blank.to_owned()), None, "{blank:?}");
+ }
+ }
+
+ #[test]
+ fn whitespace_only_ctc_decode_is_not_emitted() {
+ // A two-step decode whose only emitted class is the trailing space
+ // class that the Paddle dictionary appends.
+ let dictionary = parse_dictionary(
+ "PostProcess:\n character_dict: [a]\n",
+ DictionaryFormat::PaddleInferenceYaml,
+ )
+ .unwrap();
+ let (text, _) = crate::postprocess::ctc_decode(
+ &[0.0, 0.1, 0.9, 0.9, 0.1, 0.0],
+ &[1, 2, 3],
+ &dictionary,
+ 0,
+ )
+ .unwrap();
+ assert_eq!(text, " ");
+ assert_eq!(recognized_text(text), None);
+ }
+
#[test]
fn plain_dictionary_preserves_spaces() {
assert_eq!(