Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-perception): drop whitespace-only OCR text instead of failing the parse (#4183)
Francesco Bonaccife76ccff0417
diff --git a/libs/cua-driver/rust/crates/cua-perception/src/lib.rs b/libs/cua-driver/rust/crates/cua-perception/src/lib.rsindex 8b2b9ca007..03cc014699 100644--- a/libs/cua-driver/rust/crates/cua-perception/src/lib.rs+++ b/libs/cua-driver/rust/crates/cua-perception/src/lib.rs@@ -581,20 +581,30 @@ fn inference_result(}),);}+ // Driver validates the whole parse and rejects it if any region is empty,+ // outside the image, or carries whitespace-only text, so drop such+ // regions here instead of failing every region with them.let regions = regions.into_iter()+ .filter_map(|region| {+ let x = (region.bounds.x1.floor().max(0.0) as u32).min(width);+ let y = (region.bounds.y1.floor().max(0.0) as u32).min(height);+ let right = region.bounds.x2.ceil().min(width as f32).max(0.0) as u32;+ let bottom = region.bounds.y2.ceil().min(height as f32).max(0.0) as u32;+ let blank_text = region+ .text+ .as_deref()+ .is_some_and(|text| text.trim().is_empty());+ (right > x && bottom > y && !blank_text).then_some((region, x, y, right, bottom))+ }).enumerate()- .map(|(index, region)| {+ .map(|(index, (region, x, y, right, bottom))| {let kind = region.kind;let class_id = region.class_id;- let x = region.bounds.x1.floor().max(0.0) as u32;- let y = region.bounds.y1.floor().max(0.0) as u32;- let right = region.bounds.x2.ceil().min(width as f32) as u32;- let bottom = region.bounds.y2.ceil().min(height as f32) as u32;let mut value = json!({"id": format!("{}-{}", kind, index + 1),"kind": kind,- "bounds": { "x": x, "y": y, "width": right.saturating_sub(x), "height": bottom.saturating_sub(y) },+ "bounds": { "x": x, "y": y, "width": right - x, "height": bottom - y },"confidence": region.confidence,"interactive": false});@@ -811,6 +821,44 @@ mod tests {assert_eq!(decoded["text_geometry"], "axis_aligned_bounds");}+ #[test]+ fn degenerate_and_blank_regions_are_dropped_before_numbering() {+ let region = |kind: &'static str, x1: f32, x2: f32, text: Option<&str>| InferenceRegion {+ kind,+ bounds: Rect {+ x1,+ y1: 1.0,+ x2,+ y2: 9.0,+ },+ text: text.map(str::to_owned),+ confidence: 0.9,+ class_id: None,+ };+ let result = inference_result(+ "capture-1",+ "abc123",+ 100,+ 50,+ vec![+ region("text", 1.0, 5.0, Some(" ")),+ region("icon", 100.0, 120.0, None),+ region("text", 3.0, 3.0, Some("zero width")),+ region("text", 10.0, 30.0, Some("Submit")),+ ],+ json!({ "backend": "test" }),+ (env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")),+ );+ assert_eq!(+ result["regions"],+ json!([{+ "id": "text-1", "kind": "text",+ "bounds": { "x": 10, "y": 1, "width": 20, "height": 8 },+ "confidence": 0.8999999761581421, "text": "Submit", "interactive": false+ }])+ );+ }+#[test]fn fixture_regions_explicitly_serialize_as_non_interactive() {let result = fixture_result("capture-1", "cua-perception", "0.1.0");diff --git a/libs/cua-driver/rust/crates/cua-perception/src/runtime.rs b/libs/cua-driver/rust/crates/cua-perception/src/runtime.rsindex 5b40a22153..530f4bfcc7 100644--- a/libs/cua-driver/rust/crates/cua-perception/src/runtime.rs+++ b/libs/cua-driver/rust/crates/cua-perception/src/runtime.rs@@ -255,9 +255,12 @@ impl OnnxBackend {reason,},)?;- if text.is_empty() {+ // The recognizer dictionary includes a space class, so a crop can+ // decode to whitespace only. Driver rejects whitespace-only text, and+ // one such region would invalidate the whole parse.+ let Some(text) = recognized_text(text) else {return Ok(None);- }+ };Ok(Some(InferenceRegion {kind: "text",bounds: detection.bounds,@@ -268,6 +271,18 @@ impl OnnxBackend {}}+/// Trims recognized text and drops a crop that decoded to whitespace only.+fn recognized_text(text: String) -> Option<String> {+ let trimmed = text.trim();+ if trimmed.is_empty() {+ None+ } else if trimmed.len() == text.len() {+ Some(text)+ } else {+ Some(trimmed.to_owned())+ }+}+fn parse_dictionary(source: &str, format: DictionaryFormat) -> Result<Vec<String>, InferenceError> {let values = match format {DictionaryFormat::PlainLines => source@@ -529,6 +544,42 @@ fn run_tensor(mod tests {use super::*;+ #[test]+ fn recognized_text_trims_and_drops_whitespace_only_crops() {+ assert_eq!(+ recognized_text("Submit".to_owned()).as_deref(),+ Some("Submit")+ );+ assert_eq!(+ recognized_text(" Sign in ".to_owned()).as_deref(),+ Some("Sign in")+ );+ assert_eq!(recognized_text("a b".to_owned()).as_deref(), Some("a b"));+ for blank in ["", " ", " ", "\t", " \u{3000} "] {+ assert_eq!(recognized_text(blank.to_owned()), None, "{blank:?}");+ }+ }++ #[test]+ fn whitespace_only_ctc_decode_is_not_emitted() {+ // A two-step decode whose only emitted class is the trailing space+ // class that the Paddle dictionary appends.+ let dictionary = parse_dictionary(+ "PostProcess:\n character_dict: [a]\n",+ DictionaryFormat::PaddleInferenceYaml,+ )+ .unwrap();+ let (text, _) = crate::postprocess::ctc_decode(+ &[0.0, 0.1, 0.9, 0.9, 0.1, 0.0],+ &[1, 2, 3],+ &dictionary,+ 0,+ )+ .unwrap();+ assert_eq!(text, " ");+ assert_eq!(recognized_text(text), None);+ }+#[test]fn plain_dictionary_preserves_spaces() {assert_eq!(