YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
perf: preserve native OCR crops with run-based text detection
Ezra Ellette9fdb42102baa
diff --git a/crates/screenpipe-capture/examples/ocr_gate_ab.rs b/crates/screenpipe-capture/examples/ocr_gate_ab.rsindex 314389f96..c4f3d0d5f 100644--- a/crates/screenpipe-capture/examples/ocr_gate_ab.rs+++ b/crates/screenpipe-capture/examples/ocr_gate_ab.rs@@ -65,7 +65,7 @@ mod macos {use screenpipe_capture::{OcrDecision, OcrGate};use screenpipe_screen::monitor::{get_default_monitor, get_monitor_by_id, SafeMonitor};use screenpipe_screen::text_regions::{- detect_text_regions_bounded, image_pixel_signature, union_region, TextRegion,+ detect_text_regions, image_pixel_signature, union_region, TextRegion,};use serde_json::{json, Value};use std::collections::{HashMap, HashSet};@@ -432,7 +432,7 @@ mod macos {};let detect_for_task = detect_image.clone();let union_and_sig = tokio::task::spawn_blocking(move || {- let regions = detect_text_regions_bounded(&detect_for_task);+ let regions = detect_text_regions(&detect_for_task);let (dw, dh) = detect_for_task.dimensions();union_region(®ions, UNION_PAD_PX, dw, dh).map(|u| {let union_img = detect_for_task.crop_imm(u.x, u.y, u.width, u.height);diff --git a/crates/screenpipe-capture/src/paired_capture.rs b/crates/screenpipe-capture/src/paired_capture.rsindex 828df82f1..b1169d905 100644--- a/crates/screenpipe-capture/src/paired_capture.rs+++ b/crates/screenpipe-capture/src/paired_capture.rs@@ -23,7 +23,7 @@ use screenpipe_core::pii_removal::remove_pii;use screenpipe_db::DatabaseManager;use screenpipe_screen::snapshot_writer::SnapshotWriter;use screenpipe_screen::text_regions::{- detect_text_regions_bounded, image_pixel_signature, union_region, TextRegion,+ detect_text_regions, image_pixel_signature, union_region, TextRegion,};use screenpipe_screen::OcrGateDecision;@@ -408,8 +408,7 @@ async fn paired_capture_inner(let app_key = app_name.unwrap_or("unknown").to_lowercase();// The gated OCR pipeline (#5060) — applies to EVERY OCR// trigger, not just meetings: screenshot → crop to the app- // window → detect text on a bounded thumbnail → crop the- // native image to the padded union of the+ // window → detect text → crop to the padded union of the// detected text → pixel-compare that crop to the last// indexed one → different? OCR that same crop. One blocking// hop computes detect + union + signature.@@ -420,7 +419,7 @@ async fn paired_capture_inner(};let detect_dims = detect_image.dimensions();let union_and_sig = tokio::task::spawn_blocking(move || {- let regions = detect_text_regions_bounded(&detect_image);+ let regions = detect_text_regions(&detect_image);let (dw, dh) = detect_image.dimensions();union_region(®ions, UNION_PAD_PX, dw, dh).map(|u| {let union_img = detect_image.crop_imm(u.x, u.y, u.width, u.height);diff --git a/crates/screenpipe-screen/examples/ocr_gate_ab_windows.rs b/crates/screenpipe-screen/examples/ocr_gate_ab_windows.rsindex ff2ed902b..b0e092969 100644--- a/crates/screenpipe-screen/examples/ocr_gate_ab_windows.rs+++ b/crates/screenpipe-screen/examples/ocr_gate_ab_windows.rs@@ -56,7 +56,7 @@ mod win {use screenpipe_screen::monitor::{get_default_monitor, get_monitor_by_id};use screenpipe_screen::perform_ocr_windows;use screenpipe_screen::text_regions::{- detect_text_regions_bounded, image_pixel_signature, union_region,+ detect_text_regions, image_pixel_signature, union_region,};use std::collections::HashSet;use std::io::Write;@@ -371,7 +371,7 @@ mod win {// --- OPTIMIZED arm: detect → union crop → signature → gate. ---let d_wall = Instant::now();let o_cpu = cpu_ms();- let regions = detect_text_regions_bounded(frame);+ let regions = detect_text_regions(frame);let union = union_region(®ions, UNION_PAD_PX, fw, fh);let detect_ms = d_wall.elapsed().as_secs_f64() * 1000.0;sum.detect_ms += detect_ms;diff --git a/crates/screenpipe-screen/src/text_regions.rs b/crates/screenpipe-screen/src/text_regions.rsindex 91a6026f4..216375438 100644--- a/crates/screenpipe-screen/src/text_regions.rs+++ b/crates/screenpipe-screen/src/text_regions.rs@@ -25,9 +25,6 @@//! Cost: ~10-19ms in optimized cv2 on a 3456x2234 frame; this port is the//! same O(pixels) work. It exists so the expensive `.accurate` OCR pass//! (~400-1400ms) only runs when on-screen text actually changed.-//! Production uses `detect_text_regions_bounded`: detect on at most a 1920px-//! longest edge, scale the closing radius, and apply size filters in native-//! coordinates. The native detector remains available for fidelity probes.use image::DynamicImage;use std::hash::{DefaultHasher, Hash, Hasher};@@ -52,47 +49,6 @@ const MIN_AREA: u64 = 20;/// A component covering more than half the frame is a layout artifact/// (window border, video tile), not a text line.const MAX_AREA_FRACTION: f64 = 0.5;-/// Native-pixel radius of the horizontal close (9x1 at full resolution).-const CLOSE_RADIUS: usize = 4;--/// Bound the detector's raster work independently of the OCR input size.-const DETECT_MAX_EDGE: u32 = 1920;--/// Detect on a smaller image, returning outward-rounded native-pixel bounds.-/// Callers must still crop and compare pixels from the original image so a-/// small text edit cannot disappear from the OCR gate's change signature.-pub fn detect_text_regions_bounded(image: &DynamicImage) -> Vec<TextRegion> {- let (w, h) = (image.width(), image.height());- if w < 3 || h < 3 || w.max(h) <= DETECT_MAX_EDGE {- return detect_text_regions(image);- }- let small = image.thumbnail(DETECT_MAX_EDGE, DETECT_MAX_EDGE);- detect_text_regions_at_scale(&small, w, h)-}--fn scale_region_outward(- r: TextRegion,- from_w: u32,- from_h: u32,- to_w: u32,- to_h: u32,-) -> TextRegion {- let floor =- |v: u32, from: u32, to: u32| (u64::from(v) * u64::from(to) / u64::from(from)) as u32;- let ceil = |v: u32, from: u32, to: u32| {- (u64::from(v) * u64::from(to)).div_ceil(u64::from(from)) as u32- };- let x = floor(r.x, from_w, to_w).min(to_w);- let y = floor(r.y, from_h, to_h).min(to_h);- let right = ceil(r.x.saturating_add(r.width), from_w, to_w).min(to_w);- let bottom = ceil(r.y.saturating_add(r.height), from_h, to_h).min(to_h);- TextRegion {- x,- y,- width: right.saturating_sub(x),- height: bottom.saturating_sub(y),- }-}/// Detect text-like regions in a frame.///@@ -103,14 +59,6 @@ fn scale_region_outward(/// density gate treats "many boxes" as dense → full-frame OCR (today's/// behavior), so they can only cost, never corrupt.pub fn detect_text_regions(image: &DynamicImage) -> Vec<TextRegion> {- detect_text_regions_at_scale(image, image.width(), image.height())-}--fn detect_text_regions_at_scale(- image: &DynamicImage,- native_w: u32,- native_h: u32,-) -> Vec<TextRegion> {let (w, h) = (image.width() as usize, image.height() as usize);if w < 3 || h < 3 {return Vec::new();@@ -136,29 +84,17 @@ fn detect_text_regions_at_scale(}drop(gradient);- // Keep the gap-closing distance in native pixels. A fixed 9x1 kernel on- // a half-size image bridges twice the intended gap, merging long lines- // that the aspect filter then rejects. At least one detection pixel is- // needed to connect adjacent glyph strokes.- let radius = ((CLOSE_RADIUS as f64 * w as f64 / native_w as f64).round() as usize).max(1);- let closed = close_horizontal(&binary, w, h, radius);+ // 9x1 horizontal close (dilate then erode, radius 4): connects nearby+ // strokes into word/line blobs. Border semantics per cv2: outside is+ // black for dilate, white for erode.+ let closed = close_9x1(&binary, w, h);drop(binary);// Connected components (8-connectivity) → bounding boxes → text filter.let boxes = connected_component_boxes(&closed, w, h);- let total_area = u64::from(native_w) * u64::from(native_h);+ let total_area = (w as u64) * (h as u64);boxes.into_iter()- // Size thresholds describe native pixels, not the thumbnail. Map- // outward before filtering so small text does not fail MIN_BOX_H- // merely because we shrank the detection image.- .map(|r| {- if (w as u32, h as u32) == (native_w, native_h) {- r- } else {- scale_region_outward(r, w as u32, h as u32, native_w, native_h)- }- }).filter(|r| {if r.width < MIN_BOX_W || r.height < MIN_BOX_H {return false;@@ -380,151 +316,139 @@ fn otsu_threshold(pixels: &[u8]) -> u8 {best_t}-/// Horizontal close on a 0/1 binary image, with a radius in input pixels.-/// Out-of-bounds counts as white for erode (cv2 +inf border).-fn close_horizontal(binary: &[u8], w: usize, h: usize, radius: usize) -> Vec<u8> {- let mut dilated = vec![0u8; w * h];- for y in 0..h {- let row = &binary[y * w..(y + 1) * w];- let out = &mut dilated[y * w..(y + 1) * w];- // Sliding count of white pixels in the in-bounds window [x-radius, x+radius].- let mut count: u32 = 0;- for x in 0..radius.min(w) {- count += row[x] as u32;- }- for x in 0..w {- if x + radius < w {- count += row[x + radius] as u32;- }- out[x] = u8::from(count > 0);- if x >= radius {- count -= row[x - radius] as u32;- }- }- }+/// 9x1 morphological close on a 0/1 binary image: horizontal dilate (any+/// white within ±4 columns) then horizontal erode (all white within ±4,+/// where out-of-bounds counts as white — cv2 +inf erode border). Bridges+/// inter-glyph gaps of up to 8px into a single word/line blob.+fn close_9x1(binary: &[u8], w: usize, h: usize) -> Vec<u8> {+ const R: usize = 4;let mut closed = vec![0u8; w * h];- for y in 0..h {- let row = &dilated[y * w..(y + 1) * w];- let out = &mut closed[y * w..(y + 1) * w];- let mut count: u32 = 0;- for x in 0..radius.min(w) {- count += row[x] as u32;- }- for x in 0..w {- if x + radius < w {- count += row[x + radius] as u32;- }- // In-bounds window size at this position; out-of-bounds cells- // count as white for erosion.- let win = (x.min(radius) + 1 + radius.min(w - 1 - x)) as u32;- out[x] = u8::from(count == win);- if x >= radius {- count -= row[x - radius] as u32;+ for (row, out) in binary.chunks_exact(w).zip(closed.chunks_exact_mut(w)) {+ let mut x = 0;+ let mut pending: Option<(usize, usize)> = None;+ let fill = |out: &mut [u8], start: usize, end: usize| {+ // Dilation clips at the image border; erosion treats outside+ // pixels as white. A run within R pixels therefore reaches it.+ let start = if start <= R { 0 } else { start };+ let end = if w - end <= R { w } else { end };+ out[start..end].fill(1);+ };+ while x < w {+ let Some(offset) = row[x..].iter().position(|&p| p != 0) else {+ break;+ };+ let start = x + offset;+ x = start + 1;+ while x < w && row[x] != 0 {+ x += 1;}+ pending = Some(match pending {+ Some((left, right)) if start - right <= 2 * R => (left, x),+ Some((left, right)) => {+ fill(out, left, right);+ (start, x)+ }+ None => (start, x),+ });+ }+ if let Some((left, right)) = pending {+ fill(out, left, right);}}closed}-/// Bounding boxes of 8-connected components of white pixels, via two-pass-/// union-find labelling. Equivalent to cv2 `findContours(RETR_EXTERNAL)` +-/// `boundingRect` for this pipeline's blobs (closed strokes don't produce-/// the nested-island topology where the two differ).+/// Bounding boxes of 8-connected foreground components. Track horizontal+/// runs and their overlaps with the previous row instead of materializing+/// a full-frame label raster and scanning it again. The first encountered+/// label remains the root, preserving both native bounds and output order.fn connected_component_boxes(binary: &[u8], w: usize, h: usize) -> Vec<TextRegion> {- const NO_LABEL: u32 = u32::MAX;- let mut labels = vec![NO_LABEL; w * h];- let mut parent: Vec<u32> = Vec::new();-- fn find(parent: &mut [u32], mut i: u32) -> u32 {- while parent[i as usize] != i {- parent[i as usize] = parent[parent[i as usize] as usize];- i = parent[i as usize];+ #[derive(Clone, Copy)]+ struct Run {+ start: usize,+ end: usize, // exclusive+ label: usize,+ }+ fn find(parent: &mut [usize], mut i: usize) -> usize {+ while parent[i] != i {+ parent[i] = parent[parent[i]];+ i = parent[i];}i}- fn union(parent: &mut [u32], a: u32, b: u32) {- let (ra, rb) = (find(parent, a), find(parent, b));- if ra != rb {- parent[ra.max(rb) as usize] = ra.min(rb);- }- }-- for y in 0..h {- for x in 0..w {- if binary[y * w + x] == 0 {+ fn extend(a: &mut TextRegion, b: TextRegion) {+ let right = (a.x + a.width).max(b.x + b.width);+ let bottom = (a.y + a.height).max(b.y + b.height);+ a.x = a.x.min(b.x);+ a.y = a.y.min(b.y);+ a.width = right - a.x;+ a.height = bottom - a.y;+ }+ let mut parent = Vec::new();+ let mut boxes: Vec<TextRegion> = Vec::new();+ let mut previous: Vec<Run> = Vec::new();+ let mut current: Vec<Run> = Vec::new();+ for (y, row) in binary.chunks_exact(w).take(h).enumerate() {+ current.clear();+ let (mut x, mut above) = (0, 0);+ while x < w {+ if row[x] == 0 {+ x += 1;continue;}- // 8-connectivity: W, NW, N, NE (already-visited neighbors).- let mut neighbor_label = NO_LABEL;- let mut consider = |lbl: u32, parent: &mut Vec<u32>| {- if lbl != NO_LABEL {- if neighbor_label == NO_LABEL {- neighbor_label = lbl;- } else {- union(parent, neighbor_label, lbl);+ let start = x;+ while x < w && row[x] != 0 {+ x += 1;+ }+ let end = x;+ // Include corner-touching runs: exclusive end == start is+ // diagonal adjacency, as is previous.start == current.end.+ while above < previous.len() && previous[above].end < start {+ above += 1;+ }+ let mut label = None;+ for run in previous[above..].iter().take_while(|run| run.start <= end) {+ let root = find(&mut parent, run.label);+ label = Some(match label {+ None => root,+ Some(old) => {+ let old = find(&mut parent, old);+ let (keep, remove) = (old.min(root), old.max(root));+ if keep != remove {+ parent[remove] = keep;+ let other = boxes[remove];+ extend(&mut boxes[keep], other);+ }+ keep}- }- };- if x > 0 {- consider(labels[y * w + x - 1], &mut parent);+ });}- if y > 0 {- if x > 0 {- consider(labels[(y - 1) * w + x - 1], &mut parent);+ let bounds = TextRegion {+ x: start as u32,+ y: y as u32,+ width: (end - start) as u32,+ height: 1,+ };+ let label = match label {+ Some(label) => {+ extend(&mut boxes[label], bounds);+ label}- consider(labels[(y - 1) * w + x], &mut parent);- if x + 1 < w {- consider(labels[(y - 1) * w + x + 1], &mut parent);+ None => {+ let label = parent.len();+ parent.push(label);+ boxes.push(bounds);+ label}- }- labels[y * w + x] = if neighbor_label == NO_LABEL {- let new = parent.len() as u32;- parent.push(new);- new- } else {- neighbor_label};+ current.push(Run { start, end, label });}+ std::mem::swap(&mut previous, &mut current);}-- // Second pass: accumulate per-root extents.- #[derive(Clone, Copy)]- struct Extent {- min_x: u32,- min_y: u32,- max_x: u32,- max_y: u32,- }- let mut extents: Vec<Option<Extent>> = vec![None; parent.len()];- for y in 0..h {- for x in 0..w {- let lbl = labels[y * w + x];- if lbl == NO_LABEL {- continue;- }- let root = find(&mut parent, lbl) as usize;- let e = extents[root].get_or_insert(Extent {- min_x: x as u32,- min_y: y as u32,- max_x: x as u32,- max_y: y as u32,- });- e.min_x = e.min_x.min(x as u32);- e.min_y = e.min_y.min(y as u32);- e.max_x = e.max_x.max(x as u32);- e.max_y = e.max_y.max(y as u32);- }- }-- extents+ boxes.into_iter()- .flatten()- .map(|e| TextRegion {- x: e.min_x,- y: e.min_y,- width: e.max_x - e.min_x + 1,- height: e.max_y - e.min_y + 1,- })+ .enumerate()+ .filter_map(|(i, bounds)| (parent[i] == i).then_some(bounds)).collect()}@@ -551,115 +475,101 @@ mod tests {RgbImage::from_pixel(w, h, Rgb([235, 235, 235]))}- #[test]- fn bounded_detection_preserves_small_input_bounds() {- let mut canvas = light_canvas(400, 300);- draw_text_like_line(&mut canvas, 50, 100, 10);- let image = DynamicImage::ImageRgb8(canvas);- let native = detect_text_regions(&image);- let bounded = detect_text_regions_bounded(&image);- let coords = |rs: Vec<TextRegion>| {- rs.into_iter()- .map(|r| (r.x, r.y, r.width, r.height))- .collect::<Vec<_>>()- };- assert_eq!(coords(native), coords(bounded));- }-- #[test]- fn bounded_detection_keeps_long_lines_separated_at_word_gaps() {- let mut canvas = light_canvas(3840, 160);- // At half resolution, an unscaled closing radius joins all these- // words into one over-wide blob that MAX_ASPECT discards.- for word in 0..16 {- for stroke in 0..8 {- for dy in 0..24 {- for dx in 0..6 {- canvas.put_pixel(- 40 + word * 124 + stroke * 14 + dx,- 70 + dy,- Rgb([10, 10, 10]),- );+ // Independent flood-fill reference for the run-based component scan.+ fn flood_components(binary: &[u8], w: usize, h: usize) -> Vec<TextRegion> {+ let mut visited = vec![false; binary.len()];+ let mut boxes = Vec::new();+ for first in 0..binary.len() {+ if visited[first] || binary[first] == 0 {+ continue;+ }+ visited[first] = true;+ let mut pending = vec![first];+ let (mut min_x, mut max_x) = (first % w, first % w);+ let (mut min_y, mut max_y) = (first / w, first / w);+ while let Some(i) = pending.pop() {+ let (x, y) = (i % w, i / w);+ min_x = min_x.min(x);+ max_x = max_x.max(x);+ min_y = min_y.min(y);+ max_y = max_y.max(y);+ for yy in y.saturating_sub(1)..=(y + 1).min(h - 1) {+ for xx in x.saturating_sub(1)..=(x + 1).min(w - 1) {+ let j = yy * w + xx;+ if !visited[j] && binary[j] != 0 {+ visited[j] = true;+ pending.push(j);+ }}}}+ boxes.push(TextRegion {+ x: min_x as u32,+ y: min_y as u32,+ width: (max_x - min_x + 1) as u32,+ height: (max_y - min_y + 1) as u32,+ });}- let image = DynamicImage::ImageRgb8(canvas);- let regions = detect_text_regions_bounded(&image);- let crop =- union_region(®ions, 20, image.width(), image.height()).expect("whole line detected");- assert!(crop.x <= 40 && crop.x + crop.width >= 2004, "{crop:?}");- assert!(crop.y <= 70 && crop.y + crop.height >= 94, "{crop:?}");+ boxes}#[test]- fn bounded_detection_filters_small_text_in_native_pixels() {- let mut canvas = light_canvas(3840, 160);- for stroke in 0..12 {- for dy in 0..6 {- for dx in 0..2 {- canvas.put_pixel(40 + stroke * 4 + dx, 70 + dy, Rgb([10, 10, 10]));- }+ fn run_close_matches_dilate_then_erode_exhaustively() {+ for w in 1usize..=16 {+ for bits in 0u32..(1 << w) {+ let row: Vec<u8> = (0..w).map(|x| ((bits >> x) & 1) as u8).collect();+ let dilated: Vec<u8> = (0..w)+ .map(|x| {+ *row[x.saturating_sub(4)..=(x + 4).min(w - 1)]+ .iter()+ .max()+ .unwrap()+ })+ .collect();+ let expected: Vec<u8> = (0..w)+ .map(|x| {+ *dilated[x.saturating_sub(4)..=(x + 4).min(w - 1)]+ .iter()+ .min()+ .unwrap()+ })+ .collect();+ assert_eq!(close_9x1(&row, w, 1), expected, "width {w}, mask {bits}");}}- let image = DynamicImage::ImageRgb8(canvas);- assert!(!detect_text_regions(&image).is_empty());- let regions = detect_text_regions_bounded(&image);- let crop =- union_region(®ions, 20, image.width(), image.height()).expect("small text detected");- assert!(crop.x <= 40 && crop.x + crop.width >= 86, "{crop:?}");- assert!(crop.y <= 70 && crop.y + crop.height >= 76, "{crop:?}");}#[test]- fn scaled_bounds_round_outward_and_clamp_at_native_edges() {- let r = scale_region_outward(- TextRegion {- x: 1,- y: 2,- width: 2,- height: 3,- },- 7,- 9,- 20,- 25,- );- assert_eq!((r.x, r.y, r.width, r.height), (2, 5, 7, 9));- let edge = scale_region_outward(- TextRegion {- x: 6,- y: 8,- width: 1,- height: 1,- },- 7,- 9,- 20,- 25,- );- assert_eq!((edge.x + edge.width, edge.y + edge.height), (20, 25));+ fn run_components_match_every_four_by_four_binary_image() {+ for bits in 0u32..=u16::MAX as u32 {+ let pixels: Vec<u8> = (0..16).map(|i| ((bits >> i) & 1) as u8).collect();+ assert_eq!(+ connected_component_boxes(&pixels, 4, 4),+ flood_components(&pixels, 4, 4),+ "mask {bits}"+ );+ }}#[test]- fn bounded_detection_returns_native_coordinates_for_large_landscape_and_portrait() {- for (w, h, x, y) in [(3841, 2161, 3200, 1800), (2161, 3841, 1500, 3200)] {- let mut canvas = light_canvas(w, h);- // Large strokes remain text-like after shrinking, near a native edge.- for stroke in 0..10 {- for dx in 0..6 {- for dy in 0..24 {- canvas.put_pixel(x + stroke * 14 + dx, y + dy, Rgb([10, 10, 10]));- }- }+ fn run_components_match_sparse_dense_and_narrow_images() {+ let mut state = 0xa449_u64;+ for (w, h) in [(1, 97), (97, 1), (3, 73), (73, 3), (63, 47)] {+ for density in [0, 1, 8, 32, 64, 128, 192, 254, 255] {+ let pixels: Vec<u8> = (0..w * h)+ .map(|_| {+ state ^= state << 13;+ state ^= state >> 7;+ state ^= state << 17;+ u8::from((state & 255) < density)+ })+ .collect();+ assert_eq!(+ connected_component_boxes(&pixels, w, h),+ flood_components(&pixels, w, h),+ "{w}x{h} density {density}"+ );}- let regions = detect_text_regions_bounded(&DynamicImage::ImageRgb8(canvas));- let r = union_region(®ions, 20, w, h).expect("text crop");- assert!(- r.x <= x && r.y <= y && r.x + r.width >= x + 132 && r.y + r.height >= y + 24,- "{r:?}"- );- assert!(r.x + r.width <= w && r.y + r.height <= h);}}