Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix: scale text detection before native OCR cropping

Ezra Ellette90875ecdb9ef
diff --git a/crates/screenpipe-capture/examples/ocr_gate_ab.rs b/crates/screenpipe-capture/examples/ocr_gate_ab.rs
index c4f3d0d5f..314389f96 100644
--- a/crates/screenpipe-capture/examples/ocr_gate_ab.rs
+++ b/crates/screenpipe-capture/examples/ocr_gate_ab.rs
@@ -65,7 +65,7 @@ mod macos {
use screenpipe_capture::{OcrDecision, OcrGate};
use screenpipe_screen::monitor::{get_default_monitor, get_monitor_by_id, SafeMonitor};
use screenpipe_screen::text_regions::{
- detect_text_regions, image_pixel_signature, union_region, TextRegion,
+ detect_text_regions_bounded, image_pixel_signature, union_region, TextRegion,
};
use serde_json::{json, Value};
use std::collections::{HashMap, HashSet};
@@ -432,7 +432,7 @@ mod macos {
};
let detect_for_task = detect_image.clone();
let union_and_sig = tokio::task::spawn_blocking(move || {
- let regions = detect_text_regions(&detect_for_task);
+ let regions = detect_text_regions_bounded(&detect_for_task);
let (dw, dh) = detect_for_task.dimensions();
union_region(&regions, UNION_PAD_PX, dw, dh).map(|u| {
let union_img = detect_for_task.crop_imm(u.x, u.y, u.width, u.height);
diff --git a/crates/screenpipe-capture/src/paired_capture.rs b/crates/screenpipe-capture/src/paired_capture.rs
index b1169d905..828df82f1 100644
--- a/crates/screenpipe-capture/src/paired_capture.rs
+++ b/crates/screenpipe-capture/src/paired_capture.rs
@@ -23,7 +23,7 @@ use screenpipe_core::pii_removal::remove_pii;
use screenpipe_db::DatabaseManager;
use screenpipe_screen::snapshot_writer::SnapshotWriter;
use screenpipe_screen::text_regions::{
- detect_text_regions, image_pixel_signature, union_region, TextRegion,
+ detect_text_regions_bounded, image_pixel_signature, union_region, TextRegion,
};
use screenpipe_screen::OcrGateDecision;
@@ -408,7 +408,8 @@ async fn paired_capture_inner(
let app_key = app_name.unwrap_or("unknown").to_lowercase();
// The gated OCR pipeline (#5060) — applies to EVERY OCR
// trigger, not just meetings: screenshot → crop to the app
- // window → detect text → crop to the padded union of the
+ // window → detect text on a bounded thumbnail → crop the
+ // native image to the padded union of the
// detected text → pixel-compare that crop to the last
// indexed one → different? OCR that same crop. One blocking
// hop computes detect + union + signature.
@@ -419,7 +420,7 @@ async fn paired_capture_inner(
};
let detect_dims = detect_image.dimensions();
let union_and_sig = tokio::task::spawn_blocking(move || {
- let regions = detect_text_regions(&detect_image);
+ let regions = detect_text_regions_bounded(&detect_image);
let (dw, dh) = detect_image.dimensions();
union_region(&regions, UNION_PAD_PX, dw, dh).map(|u| {
let union_img = detect_image.crop_imm(u.x, u.y, u.width, u.height);
diff --git a/crates/screenpipe-screen/examples/ocr_gate_ab_windows.rs b/crates/screenpipe-screen/examples/ocr_gate_ab_windows.rs
index b0e092969..ff2ed902b 100644
--- a/crates/screenpipe-screen/examples/ocr_gate_ab_windows.rs
+++ b/crates/screenpipe-screen/examples/ocr_gate_ab_windows.rs
@@ -56,7 +56,7 @@ mod win {
use screenpipe_screen::monitor::{get_default_monitor, get_monitor_by_id};
use screenpipe_screen::perform_ocr_windows;
use screenpipe_screen::text_regions::{
- detect_text_regions, image_pixel_signature, union_region,
+ detect_text_regions_bounded, image_pixel_signature, union_region,
};
use std::collections::HashSet;
use std::io::Write;
@@ -371,7 +371,7 @@ mod win {
// --- OPTIMIZED arm: detect → union crop → signature → gate. ---
let d_wall = Instant::now();
let o_cpu = cpu_ms();
- let regions = detect_text_regions(frame);
+ let regions = detect_text_regions_bounded(frame);
let union = union_region(&regions, UNION_PAD_PX, fw, fh);
let detect_ms = d_wall.elapsed().as_secs_f64() * 1000.0;
sum.detect_ms += detect_ms;
diff --git a/crates/screenpipe-screen/src/text_regions.rs b/crates/screenpipe-screen/src/text_regions.rs
index a7d8d89c0..91a6026f4 100644
--- a/crates/screenpipe-screen/src/text_regions.rs
+++ b/crates/screenpipe-screen/src/text_regions.rs
@@ -25,6 +25,9 @@
//! Cost: ~10-19ms in optimized cv2 on a 3456x2234 frame; this port is the
//! same O(pixels) work. It exists so the expensive `.accurate` OCR pass
//! (~400-1400ms) only runs when on-screen text actually changed.
+//! Production uses `detect_text_regions_bounded`: detect on at most a 1920px
+//! longest edge, scale the closing radius, and apply size filters in native
+//! coordinates. The native detector remains available for fidelity probes.
use image::DynamicImage;
use std::hash::{DefaultHasher, Hash, Hasher};
@@ -49,6 +52,47 @@ const MIN_AREA: u64 = 20;
/// A component covering more than half the frame is a layout artifact
/// (window border, video tile), not a text line.
const MAX_AREA_FRACTION: f64 = 0.5;
+/// Native-pixel radius of the horizontal close (9x1 at full resolution).
+const CLOSE_RADIUS: usize = 4;
+
+/// Bound the detector's raster work independently of the OCR input size.
+const DETECT_MAX_EDGE: u32 = 1920;
+
+/// Detect on a smaller image, returning outward-rounded native-pixel bounds.
+/// Callers must still crop and compare pixels from the original image so a
+/// small text edit cannot disappear from the OCR gate's change signature.
+pub fn detect_text_regions_bounded(image: &DynamicImage) -> Vec<TextRegion> {
+ let (w, h) = (image.width(), image.height());
+ if w < 3 || h < 3 || w.max(h) <= DETECT_MAX_EDGE {
+ return detect_text_regions(image);
+ }
+ let small = image.thumbnail(DETECT_MAX_EDGE, DETECT_MAX_EDGE);
+ detect_text_regions_at_scale(&small, w, h)
+}
+
+fn scale_region_outward(
+ r: TextRegion,
+ from_w: u32,
+ from_h: u32,
+ to_w: u32,
+ to_h: u32,
+) -> TextRegion {
+ let floor =
+ |v: u32, from: u32, to: u32| (u64::from(v) * u64::from(to) / u64::from(from)) as u32;
+ let ceil = |v: u32, from: u32, to: u32| {
+ (u64::from(v) * u64::from(to)).div_ceil(u64::from(from)) as u32
+ };
+ let x = floor(r.x, from_w, to_w).min(to_w);
+ let y = floor(r.y, from_h, to_h).min(to_h);
+ let right = ceil(r.x.saturating_add(r.width), from_w, to_w).min(to_w);
+ let bottom = ceil(r.y.saturating_add(r.height), from_h, to_h).min(to_h);
+ TextRegion {
+ x,
+ y,
+ width: right.saturating_sub(x),
+ height: bottom.saturating_sub(y),
+ }
+}
/// Detect text-like regions in a frame.
///
@@ -59,6 +103,14 @@ const MAX_AREA_FRACTION: f64 = 0.5;
/// density gate treats "many boxes" as dense → full-frame OCR (today's
/// behavior), so they can only cost, never corrupt.
pub fn detect_text_regions(image: &DynamicImage) -> Vec<TextRegion> {
+ detect_text_regions_at_scale(image, image.width(), image.height())
+}
+
+fn detect_text_regions_at_scale(
+ image: &DynamicImage,
+ native_w: u32,
+ native_h: u32,
+) -> Vec<TextRegion> {
let (w, h) = (image.width() as usize, image.height() as usize);
if w < 3 || h < 3 {
return Vec::new();
@@ -84,17 +136,29 @@ pub fn detect_text_regions(image: &DynamicImage) -> Vec<TextRegion> {
}
drop(gradient);
- // 9x1 horizontal close (dilate then erode, radius 4): connects nearby
- // strokes into word/line blobs. Border semantics per cv2: outside is
- // black for dilate, white for erode.
- let closed = close_9x1(&binary, w, h);
+ // Keep the gap-closing distance in native pixels. A fixed 9x1 kernel on
+ // a half-size image bridges twice the intended gap, merging long lines
+ // that the aspect filter then rejects. At least one detection pixel is
+ // needed to connect adjacent glyph strokes.
+ let radius = ((CLOSE_RADIUS as f64 * w as f64 / native_w as f64).round() as usize).max(1);
+ let closed = close_horizontal(&binary, w, h, radius);
drop(binary);
// Connected components (8-connectivity) → bounding boxes → text filter.
let boxes = connected_component_boxes(&closed, w, h);
- let total_area = (w as u64) * (h as u64);
+ let total_area = u64::from(native_w) * u64::from(native_h);
boxes
.into_iter()
+ // Size thresholds describe native pixels, not the thumbnail. Map
+ // outward before filtering so small text does not fail MIN_BOX_H
+ // merely because we shrank the detection image.
+ .map(|r| {
+ if (w as u32, h as u32) == (native_w, native_h) {
+ r
+ } else {
+ scale_region_outward(r, w as u32, h as u32, native_w, native_h)
+ }
+ })
.filter(|r| {
if r.width < MIN_BOX_W || r.height < MIN_BOX_H {
return false;
@@ -316,28 +380,25 @@ fn otsu_threshold(pixels: &[u8]) -> u8 {
best_t
}
-/// 9x1 morphological close on a 0/1 binary image: horizontal dilate (any
-/// white within ±4 columns) then horizontal erode (all white within ±4,
-/// where out-of-bounds counts as white — cv2 +inf erode border). Bridges
-/// inter-glyph gaps of up to 8px into a single word/line blob.
-fn close_9x1(binary: &[u8], w: usize, h: usize) -> Vec<u8> {
- const R: usize = 4;
+/// Horizontal close on a 0/1 binary image, with a radius in input pixels.
+/// Out-of-bounds counts as white for erode (cv2 +inf border).
+fn close_horizontal(binary: &[u8], w: usize, h: usize, radius: usize) -> Vec<u8> {
let mut dilated = vec![0u8; w * h];
for y in 0..h {
let row = &binary[y * w..(y + 1) * w];
let out = &mut dilated[y * w..(y + 1) * w];
- // Sliding count of white pixels in the in-bounds window [x-R, x+R].
+ // Sliding count of white pixels in the in-bounds window [x-radius, x+radius].
let mut count: u32 = 0;
- for x in 0..R.min(w) {
+ for x in 0..radius.min(w) {
count += row[x] as u32;
}
for x in 0..w {
- if x + R < w {
- count += row[x + R] as u32;
+ if x + radius < w {
+ count += row[x + radius] as u32;
}
out[x] = u8::from(count > 0);
- if x >= R {
- count -= row[x - R] as u32;
+ if x >= radius {
+ count -= row[x - radius] as u32;
}
}
}
@@ -346,19 +407,19 @@ fn close_9x1(binary: &[u8], w: usize, h: usize) -> Vec<u8> {
let row = &dilated[y * w..(y + 1) * w];
let out = &mut closed[y * w..(y + 1) * w];
let mut count: u32 = 0;
- for x in 0..R.min(w) {
+ for x in 0..radius.min(w) {
count += row[x] as u32;
}
for x in 0..w {
- if x + R < w {
- count += row[x + R] as u32;
+ if x + radius < w {
+ count += row[x + radius] as u32;
}
// In-bounds window size at this position; out-of-bounds cells
// count as white for erosion.
- let win = (x.min(R) + 1 + R.min(w - 1 - x)) as u32;
+ let win = (x.min(radius) + 1 + radius.min(w - 1 - x)) as u32;
out[x] = u8::from(count == win);
- if x >= R {
- count -= row[x - R] as u32;
+ if x >= radius {
+ count -= row[x - radius] as u32;
}
}
}
@@ -490,6 +551,118 @@ mod tests {
RgbImage::from_pixel(w, h, Rgb([235, 235, 235]))
}
+ #[test]
+ fn bounded_detection_preserves_small_input_bounds() {
+ let mut canvas = light_canvas(400, 300);
+ draw_text_like_line(&mut canvas, 50, 100, 10);
+ let image = DynamicImage::ImageRgb8(canvas);
+ let native = detect_text_regions(&image);
+ let bounded = detect_text_regions_bounded(&image);
+ let coords = |rs: Vec<TextRegion>| {
+ rs.into_iter()
+ .map(|r| (r.x, r.y, r.width, r.height))
+ .collect::<Vec<_>>()
+ };
+ assert_eq!(coords(native), coords(bounded));
+ }
+
+ #[test]
+ fn bounded_detection_keeps_long_lines_separated_at_word_gaps() {
+ let mut canvas = light_canvas(3840, 160);
+ // At half resolution, an unscaled closing radius joins all these
+ // words into one over-wide blob that MAX_ASPECT discards.
+ for word in 0..16 {
+ for stroke in 0..8 {
+ for dy in 0..24 {
+ for dx in 0..6 {
+ canvas.put_pixel(
+ 40 + word * 124 + stroke * 14 + dx,
+ 70 + dy,
+ Rgb([10, 10, 10]),
+ );
+ }
+ }
+ }
+ }
+ let image = DynamicImage::ImageRgb8(canvas);
+ let regions = detect_text_regions_bounded(&image);
+ let crop =
+ union_region(&regions, 20, image.width(), image.height()).expect("whole line detected");
+ assert!(crop.x <= 40 && crop.x + crop.width >= 2004, "{crop:?}");
+ assert!(crop.y <= 70 && crop.y + crop.height >= 94, "{crop:?}");
+ }
+
+ #[test]
+ fn bounded_detection_filters_small_text_in_native_pixels() {
+ let mut canvas = light_canvas(3840, 160);
+ for stroke in 0..12 {
+ for dy in 0..6 {
+ for dx in 0..2 {
+ canvas.put_pixel(40 + stroke * 4 + dx, 70 + dy, Rgb([10, 10, 10]));
+ }
+ }
+ }
+ let image = DynamicImage::ImageRgb8(canvas);
+ assert!(!detect_text_regions(&image).is_empty());
+ let regions = detect_text_regions_bounded(&image);
+ let crop =
+ union_region(&regions, 20, image.width(), image.height()).expect("small text detected");
+ assert!(crop.x <= 40 && crop.x + crop.width >= 86, "{crop:?}");
+ assert!(crop.y <= 70 && crop.y + crop.height >= 76, "{crop:?}");
+ }
+
+ #[test]
+ fn scaled_bounds_round_outward_and_clamp_at_native_edges() {
+ let r = scale_region_outward(
+ TextRegion {
+ x: 1,
+ y: 2,
+ width: 2,
+ height: 3,
+ },
+ 7,
+ 9,
+ 20,
+ 25,
+ );
+ assert_eq!((r.x, r.y, r.width, r.height), (2, 5, 7, 9));
+ let edge = scale_region_outward(
+ TextRegion {
+ x: 6,
+ y: 8,
+ width: 1,
+ height: 1,
+ },
+ 7,
+ 9,
+ 20,
+ 25,
+ );
+ assert_eq!((edge.x + edge.width, edge.y + edge.height), (20, 25));
+ }
+
+ #[test]
+ fn bounded_detection_returns_native_coordinates_for_large_landscape_and_portrait() {
+ for (w, h, x, y) in [(3841, 2161, 3200, 1800), (2161, 3841, 1500, 3200)] {
+ let mut canvas = light_canvas(w, h);
+ // Large strokes remain text-like after shrinking, near a native edge.
+ for stroke in 0..10 {
+ for dx in 0..6 {
+ for dy in 0..24 {
+ canvas.put_pixel(x + stroke * 14 + dx, y + dy, Rgb([10, 10, 10]));
+ }
+ }
+ }
+ let regions = detect_text_regions_bounded(&DynamicImage::ImageRgb8(canvas));
+ let r = union_region(&regions, 20, w, h).expect("text crop");
+ assert!(
+ r.x <= x && r.y <= y && r.x + r.width >= x + 132 && r.y + r.height >= y + 24,
+ "{r:?}"
+ );
+ assert!(r.x + r.width <= w && r.y + r.height <= h);
+ }
+ }
+
#[test]
fn blank_image_has_no_regions() {
let img = DynamicImage::ImageRgb8(light_canvas(400, 300));