YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(capture): let macOS OCR read full-resolution frames on wide displays
Manas Raghuwanshi110ee0470de5
diff --git a/apps/screenpipe-app-tauri/lib/utils/tauri.ts b/apps/screenpipe-app-tauri/lib/utils/tauri.tsindex bb13a2233..9f3fc52ca 100644--- a/apps/screenpipe-app-tauri/lib/utils/tauri.ts+++ b/apps/screenpipe-app-tauri/lib/utils/tauri.ts@@ -3933,6 +3933,13 @@ useAllMonitors: boolean;* Video quality preset: "low", "balanced", "high", "max".*/videoQuality: string;+/**+ * Max width (px) of the macOS screen capture that OCR reads. 0 = native;+ * nonzero values below 1280 are raised to 1280. Stored snapshots use the+ * smaller of this width and the `videoQuality` width. Other platforms+ * always capture native.+ */+captureMaxWidth?: number;/*** Maximum width for stored snapshots. Images wider than this are downscaled* (preserving aspect ratio) before JPEG encoding. 0 = no limit (store atdiff --git a/crates/screenpipe-config/src/recording.rs b/crates/screenpipe-config/src/recording.rsindex 1455e6ed0..d0074e265 100644--- a/crates/screenpipe-config/src/recording.rs+++ b/crates/screenpipe-config/src/recording.rs@@ -446,6 +446,13 @@ pub struct RecordingSettings {#[serde(rename = "videoQuality")]pub video_quality: String,+ /// Max width (px) of the macOS screen capture that OCR reads. 0 = native;+ /// nonzero values below 1280 are raised to 1280. Stored snapshots use the+ /// smaller of this width and the `videoQuality` width. Other platforms+ /// always capture native.+ #[serde(rename = "captureMaxWidth", default)]+ pub capture_max_width: u32,+/// Maximum width for stored snapshots. Images wider than this are downscaled/// (preserving aspect ratio) before JPEG encoding. 0 = no limit (store at/// native resolution). Default: 1920.@@ -938,6 +945,7 @@ impl Default for RecordingSettings {monitor_ids: vec![],use_all_monitors: true,video_quality: "balanced".to_string(),+ capture_max_width: 0,max_snapshot_width: default_max_snapshot_width(),disable_snapshot_compaction: false,disable_meeting_detector: false,diff --git a/crates/screenpipe-engine/src/cli/mod.rs b/crates/screenpipe-engine/src/cli/mod.rsindex 95735f1e1..9d9a8e444 100644--- a/crates/screenpipe-engine/src/cli/mod.rs+++ b/crates/screenpipe-engine/src/cli/mod.rs@@ -703,6 +703,14 @@ pub struct RecordArgs {#[arg(long, default_value = "balanced")]pub video_quality: String,+ /// Max width in pixels of the macOS screen capture that OCR reads.+ /// 0 = native; nonzero values below 1280 are raised to 1280. Stored+ /// screenshots use the smaller of this width and the --video-quality+ /// width. Lower it to trade OCR legibility for less GPU work on wide+ /// displays. Other platforms always capture native.+ #[arg(long, default_value_t = 0)]+ pub capture_max_width: u32,+/// Keep the computer awake while screenpipe is running.#[arg(long, default_value_t = false)]pub keep_computer_awake: bool,@@ -933,6 +941,7 @@ pub struct RecordArgSources {pub transcription_mode: bool,pub disable_telemetry: bool,pub video_quality: bool,+ pub capture_max_width: bool,pub keep_computer_awake: bool,pub pause_on_drm_content: bool,pub disable_clipboard_capture: bool,@@ -1002,6 +1011,7 @@ impl RecordArgSources {transcription_mode: from_command_line(record, "transcription_mode"),disable_telemetry: from_command_line(record, "disable_telemetry"),video_quality: from_command_line(record, "video_quality"),+ capture_max_width: from_command_line(record, "capture_max_width"),keep_computer_awake: from_command_line(record, "keep_computer_awake"),pause_on_drm_content: from_command_line(record, "pause_on_drm_content"),disable_clipboard_capture: from_command_line(record, "disable_clipboard_capture"),@@ -1057,6 +1067,7 @@ impl RecordArgSources {|| self.transcription_mode|| self.disable_telemetry|| self.video_quality+ || self.capture_max_width|| self.keep_computer_awake|| self.pause_on_drm_content|| self.disable_clipboard_capture@@ -1249,6 +1260,7 @@ impl RecordArgs {.collect(),deepgram_api_key: self.deepgram_api_key.clone().unwrap_or_default(),video_quality: self.video_quality.clone(),+ capture_max_width: self.capture_max_width,disable_snapshot_compaction: self.disable_snapshot_compaction,disable_meeting_detector: self.disable_meeting_detector,idle_capture_interval_ms: self.idle_capture_interval_ms,@@ -1581,6 +1593,9 @@ impl RecordArgs {if sources.video_quality {settings.video_quality = self.video_quality.clone();}+ if sources.capture_max_width {+ settings.capture_max_width = self.capture_max_width;+ }if sources.keep_computer_awake {settings.keep_computer_awake = self.keep_computer_awake;}@@ -2718,6 +2733,7 @@ mod tests {disable_clipboard_capture: false,disable_keyboard_capture: false,video_quality: "max".to_string(),+ capture_max_width: 2560,use_all_monitors: false,monitor_ids: vec!["42".to_string()],..Default::default()@@ -2728,6 +2744,20 @@ mod tests {assert!(!sources.has_recording_override());}+ #[test]+ fn explicit_capture_max_width_overrides_saved_value() {+ let argv = ["screenpipe", "record", "--capture-max-width", "1920"];+ let args = record_args(argv);+ let sources = record_sources(argv);+ let mut settings = screenpipe_config::RecordingSettings {+ capture_max_width: 2560,+ ..Default::default()+ };+ args.apply_defaults_and_overrides(&mut settings, None, &sources);+ assert_eq!(settings.capture_max_width, 1920);+ assert!(sources.has_recording_override());+ }+#[test]fn test_fresh_hardware_defaults_survive_absent_cli_flags() {use screenpipe_config::DeviceTier;diff --git a/crates/screenpipe-engine/src/hd_recorder.rs b/crates/screenpipe-engine/src/hd_recorder.rsindex 6da750ba7..729aad78e 100644--- a/crates/screenpipe-engine/src/hd_recorder.rs+++ b/crates/screenpipe-engine/src/hd_recorder.rs@@ -74,6 +74,9 @@ pub struct HdRecorderConfig {pub ignore_incognito_windows: bool,/// Let the incognito detector use browser-native APIs (macOS Automation).pub enhanced_incognito_detection: bool,+ /// Stored snapshot width from `videoQuality` (0 = native). Bounds the HD+ /// stream; independent of the OCR capture width.+ pub snapshot_max_width: u32,}/// Per-monitor HD recorder loop. Idles until an HD session is active, then@@ -395,9 +398,12 @@ mod macos {// starts the stream, so run it on a blocking thread.let hd = {let m = monitor.clone();- tokio::task::spawn_blocking(move || m.start_hd_capture(fps, &excluded))- .await- .context("hd capture spawn_blocking join")??+ let snapshot_max_width = config.snapshot_max_width;+ tokio::task::spawn_blocking(move || {+ m.start_hd_capture(fps, &excluded, snapshot_max_width)+ })+ .await+ .context("hd capture spawn_blocking join")??};let actual_fps = hd.fps.max(1);let (hd_w, hd_h) = (hd.width, hd.height);diff --git a/crates/screenpipe-engine/src/recording_config.rs b/crates/screenpipe-engine/src/recording_config.rsindex ad083e961..e1797a843 100644--- a/crates/screenpipe-engine/src/recording_config.rs+++ b/crates/screenpipe-engine/src/recording_config.rs@@ -18,6 +18,11 @@ use std::time::Duration;use crate::vision_manager::VisionManagerConfig;+/// Smallest non-native capture width. Narrower captures make OCR unreadable,+/// and since stored snapshots are never upscaled they would also shrink every+/// stored frame below the `low` preset (1280). Smaller values are raised.+const MIN_CAPTURE_MAX_WIDTH: u32 = 1280;+/// Unified recording configuration used by both the CLI binary and the Tauri embedded server./// Replaces the former `EmbeddedServerConfig` and eliminates duplicate field mapping.#[derive(Clone, Debug)]@@ -162,6 +167,9 @@ pub struct RecordingConfig {/// Video quality preset controlling JPEG quality during frame extraction./// Values: "low", "balanced", "high", "max". Default: "balanced".pub video_quality: String,+ /// Max width (px) of the macOS screen capture that OCR reads. 0 = native,+ /// otherwise at least `MIN_CAPTURE_MAX_WIDTH`.+ pub capture_max_width: u32,// Miscpub use_chinese_mirror: bool,@@ -385,6 +393,10 @@ impl RecordingConfig {openai_compatible_raw_audio: settings.openai_compatible_raw_audio,user_name: settings.user_name.clone(),video_quality: settings.video_quality.clone(),+ capture_max_width: match settings.capture_max_width {+ 0 => 0,+ width => width.max(MIN_CAPTURE_MAX_WIDTH),+ },use_chinese_mirror: settings.use_chinese_mirror,analytics_enabled: settings.analytics_enabled,analytics_id: settings.analytics_id.clone(),@@ -547,6 +559,7 @@ impl RecordingConfig {pause_on_drm_content: self.pause_on_drm_content,languages: self.languages.clone(),video_quality: self.video_quality.clone(),+ capture_max_width: self.capture_max_width,disable_screenshots: self.disable_screenshots,enable_semantic_context: self.enable_semantic_context,semantic_context_mode: self.semantic_context_mode,@@ -802,6 +815,7 @@ mod tests {monitor_ids: vec!["MONITOR-1".to_string()],use_all_monitors: false,video_quality: "high".to_string(),+ capture_max_width: 2560,idle_capture_interval_ms: Some(2_000),visual_check_interval_ms: Some(350),visual_change_threshold: Some(0.18),@@ -830,6 +844,7 @@ mod tests {assert!(vision.enhanced_incognito_detection);assert!(vision.pause_on_drm_content);assert_eq!(vision.video_quality, "high");+ assert_eq!(vision.capture_max_width, 2560);assert!(!vision.disable_screenshots);assert_eq!(vision.idle_capture_interval_ms, Some(2_000));assert_eq!(vision.visual_check_interval_ms, Some(350));@@ -839,6 +854,40 @@ mod tests {assert_eq!(vision.capture_on_clipboard, Some(true));}+ /// OCR reads the captured frame, so no storage preset may shrink the+ /// capture by default (#7393).+ #[test]+ fn capture_defaults_to_native_for_every_video_quality() {+ for quality in ["low", "balanced", "high", "max"] {+ let settings = screenpipe_config::RecordingSettings {+ video_quality: quality.to_string(),+ ..Default::default()+ };+ let vision = build(&settings).to_vision_manager_config(+ String::new(),+ std::sync::Arc::new(PipelineMetrics::new()),+ );+ assert_eq!(vision.capture_max_width, 0, "videoQuality={quality}");+ }+ }++ /// A tiny cap would store 1-pixel-wide frames and lose the recording.+ #[test]+ fn capture_max_width_below_floor_is_raised() {+ for (configured, effective) in [(0, 0), (1, 1280), (1279, 1280), (1280, 1280), (2560, 2560)]+ {+ let settings = screenpipe_config::RecordingSettings {+ capture_max_width: configured,+ ..Default::default()+ };+ assert_eq!(+ build(&settings).capture_max_width,+ effective,+ "captureMaxWidth={configured}"+ );+ }+ }+fn langs(items: &[&str]) -> Vec<String> {items.iter().map(|s| s.to_string()).collect()}diff --git a/crates/screenpipe-engine/src/vision_manager/manager.rs b/crates/screenpipe-engine/src/vision_manager/manager.rsindex cac6dada5..f3f729ed6 100644--- a/crates/screenpipe-engine/src/vision_manager/manager.rs+++ b/crates/screenpipe-engine/src/vision_manager/manager.rs@@ -59,6 +59,9 @@ pub struct VisionManagerConfig {/// snapshot max width via `screenpipe_core::video::*`. Values: "low",/// "balanced" (default), "high", "max".pub video_quality: String,+ /// Max width (px) of the macOS screen capture that OCR reads. 0 = native.+ /// Independent of `video_quality`, which only sizes what is stored.+ pub capture_max_width: u32,/// Skip screenshot pixels/JPEG/OCR while keeping accessibility-tree capture.pub disable_screenshots: bool,/// Enable the bounded semantic projection worker. Off by default.@@ -627,15 +630,13 @@ impl VisionManager {max_snapshot_width,));- // Cap the macOS SCK capture stream to the same width as the snapshot- // writer. The GPU downscales before replayd delivers the framebuffer,- // saving WindowServer composite + readback cost without affecting- // anything that wasn't going to be downsized in user space anyway.- // Text extraction is primarily a11y-tree-driven (unchanged) and OCR- // runs only as a fallback; both see the same image they'd see after- // the snapshot-writer downscale.+ // OCR reads the captured frame, not the stored JPEG, so the capture+ // cap decides OCR legibility. Never set it from `max_snapshot_width`+ // or `videoQuality`: capping capture at the stored width garbled+ // small text on wide displays (#7393). The snapshot writer shrinks+ // its own copy.#[cfg(target_os = "macos")]- screenpipe_screen::monitor::set_sck_capture_max_width(max_snapshot_width);+ screenpipe_screen::monitor::set_sck_capture_max_width(self.config.capture_max_width);// Create activity feed for this monitorlet activity_feed = ActivityFeed::new();@@ -726,6 +727,7 @@ impl VisionManager {included_urls: self.config.included_urls.clone(),ignore_incognito_windows: self.config.ignore_incognito_windows,enhanced_incognito_detection: self.config.enhanced_incognito_detection,+ snapshot_max_width: max_snapshot_width,};let hd_handle = self.vision_handle@@ -1035,6 +1037,7 @@ mod tests {pause_on_drm_content: false,languages: vec![Language::English],video_quality: "balanced".to_string(),+ capture_max_width: 0,disable_screenshots: false,enable_semantic_context,semantic_context_mode: SemanticContextMode::Memory,diff --git a/crates/screenpipe-screen/src/monitor/macos.rs b/crates/screenpipe-screen/src/monitor/macos.rsindex 35a3600c0..0d88e22b7 100644--- a/crates/screenpipe-screen/src/monitor/macos.rs+++ b/crates/screenpipe-screen/src/monitor/macos.rs@@ -64,22 +64,18 @@ static SCK_E2E_LOOKUP_HANG_ARMED: std::sync::atomic::AtomicBool =static SCK_E2E_CAPTURE_HANG_INJECTED: std::sync::atomic::AtomicBool =std::sync::atomic::AtomicBool::new(false);-/// Optional cap on captured width for the macOS SCK stream. The GPU-/// downscales to fit before `replayd` delivers the framebuffer, so+/// Optional cap on captured width for the macOS SCK screenshot stream. The+/// GPU downscales to fit before `replayd` delivers the framebuffer, so/// WindowServer composites a smaller surface. `0` = no cap (capture at-/// native, the previous default).+/// native, the default).///-/// Set by the vision manager from the user's `video_quality` setting via-/// `video_quality_to_max_snapshot_width` so the captured resolution-/// matches the resolution the snapshot writer would downscale to anyway.-/// Coupling these means screenpipe captures exactly the pixels it stores —-/// no wasted GPU readback at native res just to throw it away in user-/// space before JPEG encoding.+/// This is the resolution OCR reads. Storage is shrunk separately by+/// `SnapshotWriter`, so do not set this from storage settings such as+/// `video_quality`: doing so garbled small text on wide displays (#7393).static SCK_CAPTURE_MAX_WIDTH: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);-/// Set the SCK capture-width cap. `0` = no cap. Reads the user's-/// `video_quality` setting via the snapshot-width mapping; called by the-/// vision manager when capture starts and when settings change.+/// Set the SCK capture-width cap. `0` = no cap. Called by the vision manager+/// from the user's `captureMaxWidth` setting when capture starts.pub fn set_sck_capture_max_width(max_width: u32) {SCK_CAPTURE_MAX_WIDTH.store(max_width, std::sync::atomic::Ordering::Relaxed);}@@ -1045,8 +1041,8 @@ fn hd_scaled_dims(src_w: u32, src_h: u32, max_width: u32) -> (u32, u32) {impl SafeMonitor {/// Start a dedicated high-fps HD capture stream for this monitor at `fps`,- /// honoring the screenshot resolution cap while independently bounding the- /// live meeting stream to 1920px wide.+ /// honoring the stored snapshot width (`snapshot_max_width`, 0 = native)+ /// while independently bounding the live meeting stream to 1920px wide.////// Returns a live [`HdCapture`]: drain `frames` for RGBA frames, drop/// `stream` to stop. This opens a SECOND ScreenCaptureKit stream alongside@@ -1054,11 +1050,16 @@ impl SafeMonitor {/// path. `excluded_window_ids` are excluded at the OS level — ignored //// private windows never reach the recorder. Blocks briefly while the/// stream starts; call from a blocking context.- pub fn start_hd_capture(&self, fps: u32, excluded_window_ids: &[u32]) -> Result<HdCapture> {+ pub fn start_hd_capture(+ &self,+ fps: u32,+ excluded_window_ids: &[u32],+ snapshot_max_width: u32,+ ) -> Result<HdCapture> {let (width, height) = hd_scaled_dims(self.monitor_data.width,self.monitor_data.height,- hd_capture_max_width(sck_capture_max_width()),+ hd_capture_max_width(snapshot_max_width),);let (stream, frames) =sck_rs::start_hd_capture(self.monitor_id, width, height, fps, excluded_window_ids)diff --git a/crates/screenpipe-screen/src/snapshot_writer.rs b/crates/screenpipe-screen/src/snapshot_writer.rsindex bc48062a1..4e5b9e6ca 100644--- a/crates/screenpipe-screen/src/snapshot_writer.rs+++ b/crates/screenpipe-screen/src/snapshot_writer.rs@@ -177,6 +177,20 @@ mod tests {assert_eq!(loaded.height(), 1080);}+ /// Capture can be wider than storage (#7393), so the writer itself must+ /// cap the stored JPEG.+ #[test]+ fn test_write_downscales_to_max_width() {+ let tmp = TempDir::new().unwrap();+ let writer = SnapshotWriter::new(tmp.path(), 80, 1920);+ let img = test_image(4000, 2000);++ let path = writer.write(&img, Utc::now(), 0).unwrap();++ let loaded = image::open(&path).unwrap();+ assert_eq!((loaded.width(), loaded.height()), (1920, 960));+ }+#[test]fn test_write_creates_date_directory() {let tmp = TempDir::new().unwrap();diff --git a/docs/TESTING.md b/docs/TESTING.mdindex 082d46c81..a43f24558 100644--- a/docs/TESTING.md+++ b/docs/TESTING.md@@ -242,6 +242,7 @@ commits: `6dd5d98e`, `831ad258`- [ ] **identical frames skipped** — check logs for hash match frequency on idle monitors. should be >80% skip rate.- [ ] **ultrawide monitor (3440x1440+)** — OCR works correctly. no distortion in change detection. text at edges is captured.- [ ] **4K monitor** — OCR works. frame comparison doesn't timeout or spike CPU.+- [ ] **OCR reads full-size frames on wide displays (macOS)** — On a display wider than 1920 points with default settings (`balanced`), open a terminal with small text and search for a word from it: OCR finds it. Stored JPEGs stay at the `videoQuality` width (1920 for `balanced`), and meeting HD video width is unchanged. With `balanced`, `--capture-max-width 1920` restores the old capture size and cost. Capture width must never be derived from `videoQuality` (#7393).- [ ] **high refresh rate (120Hz+)** — app respects its own FPS setting (0.5 default), not the display refresh rate.- [ ] **very fast content changes** — scroll quickly through a document. OCR captures content, no crashes from buffer overflows.- [ ] **corrupt pixel buffer** — sck-rs handles corrupt ScreenCaptureKit buffers gracefully (no SIGABRT). fixed in `831ad258`.