Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(capture): let macOS OCR read full-resolution frames on wide displays

Manas Raghuwanshi110ee0470de5
diff --git a/apps/screenpipe-app-tauri/lib/utils/tauri.ts b/apps/screenpipe-app-tauri/lib/utils/tauri.ts
index bb13a2233..9f3fc52ca 100644
--- a/apps/screenpipe-app-tauri/lib/utils/tauri.ts
+++ b/apps/screenpipe-app-tauri/lib/utils/tauri.ts
@@ -3933,6 +3933,13 @@ useAllMonitors: boolean;
* Video quality preset: "low", "balanced", "high", "max".
*/
videoQuality: string;
+/**
+ * Max width (px) of the macOS screen capture that OCR reads. 0 = native;
+ * nonzero values below 1280 are raised to 1280. Stored snapshots use the
+ * smaller of this width and the `videoQuality` width. Other platforms
+ * always capture native.
+ */
+captureMaxWidth?: number;
/**
* Maximum width for stored snapshots. Images wider than this are downscaled
* (preserving aspect ratio) before JPEG encoding. 0 = no limit (store at
diff --git a/crates/screenpipe-config/src/recording.rs b/crates/screenpipe-config/src/recording.rs
index 1455e6ed0..d0074e265 100644
--- a/crates/screenpipe-config/src/recording.rs
+++ b/crates/screenpipe-config/src/recording.rs
@@ -446,6 +446,13 @@ pub struct RecordingSettings {
#[serde(rename = "videoQuality")]
pub video_quality: String,
+ /// Max width (px) of the macOS screen capture that OCR reads. 0 = native;
+ /// nonzero values below 1280 are raised to 1280. Stored snapshots use the
+ /// smaller of this width and the `videoQuality` width. Other platforms
+ /// always capture native.
+ #[serde(rename = "captureMaxWidth", default)]
+ pub capture_max_width: u32,
+
/// Maximum width for stored snapshots. Images wider than this are downscaled
/// (preserving aspect ratio) before JPEG encoding. 0 = no limit (store at
/// native resolution). Default: 1920.
@@ -938,6 +945,7 @@ impl Default for RecordingSettings {
monitor_ids: vec![],
use_all_monitors: true,
video_quality: "balanced".to_string(),
+ capture_max_width: 0,
max_snapshot_width: default_max_snapshot_width(),
disable_snapshot_compaction: false,
disable_meeting_detector: false,
diff --git a/crates/screenpipe-engine/src/cli/mod.rs b/crates/screenpipe-engine/src/cli/mod.rs
index 95735f1e1..9d9a8e444 100644
--- a/crates/screenpipe-engine/src/cli/mod.rs
+++ b/crates/screenpipe-engine/src/cli/mod.rs
@@ -703,6 +703,14 @@ pub struct RecordArgs {
#[arg(long, default_value = "balanced")]
pub video_quality: String,
+ /// Max width in pixels of the macOS screen capture that OCR reads.
+ /// 0 = native; nonzero values below 1280 are raised to 1280. Stored
+ /// screenshots use the smaller of this width and the --video-quality
+ /// width. Lower it to trade OCR legibility for less GPU work on wide
+ /// displays. Other platforms always capture native.
+ #[arg(long, default_value_t = 0)]
+ pub capture_max_width: u32,
+
/// Keep the computer awake while screenpipe is running.
#[arg(long, default_value_t = false)]
pub keep_computer_awake: bool,
@@ -933,6 +941,7 @@ pub struct RecordArgSources {
pub transcription_mode: bool,
pub disable_telemetry: bool,
pub video_quality: bool,
+ pub capture_max_width: bool,
pub keep_computer_awake: bool,
pub pause_on_drm_content: bool,
pub disable_clipboard_capture: bool,
@@ -1002,6 +1011,7 @@ impl RecordArgSources {
transcription_mode: from_command_line(record, "transcription_mode"),
disable_telemetry: from_command_line(record, "disable_telemetry"),
video_quality: from_command_line(record, "video_quality"),
+ capture_max_width: from_command_line(record, "capture_max_width"),
keep_computer_awake: from_command_line(record, "keep_computer_awake"),
pause_on_drm_content: from_command_line(record, "pause_on_drm_content"),
disable_clipboard_capture: from_command_line(record, "disable_clipboard_capture"),
@@ -1057,6 +1067,7 @@ impl RecordArgSources {
|| self.transcription_mode
|| self.disable_telemetry
|| self.video_quality
+ || self.capture_max_width
|| self.keep_computer_awake
|| self.pause_on_drm_content
|| self.disable_clipboard_capture
@@ -1249,6 +1260,7 @@ impl RecordArgs {
.collect(),
deepgram_api_key: self.deepgram_api_key.clone().unwrap_or_default(),
video_quality: self.video_quality.clone(),
+ capture_max_width: self.capture_max_width,
disable_snapshot_compaction: self.disable_snapshot_compaction,
disable_meeting_detector: self.disable_meeting_detector,
idle_capture_interval_ms: self.idle_capture_interval_ms,
@@ -1581,6 +1593,9 @@ impl RecordArgs {
if sources.video_quality {
settings.video_quality = self.video_quality.clone();
}
+ if sources.capture_max_width {
+ settings.capture_max_width = self.capture_max_width;
+ }
if sources.keep_computer_awake {
settings.keep_computer_awake = self.keep_computer_awake;
}
@@ -2718,6 +2733,7 @@ mod tests {
disable_clipboard_capture: false,
disable_keyboard_capture: false,
video_quality: "max".to_string(),
+ capture_max_width: 2560,
use_all_monitors: false,
monitor_ids: vec!["42".to_string()],
..Default::default()
@@ -2728,6 +2744,20 @@ mod tests {
assert!(!sources.has_recording_override());
}
+ #[test]
+ fn explicit_capture_max_width_overrides_saved_value() {
+ let argv = ["screenpipe", "record", "--capture-max-width", "1920"];
+ let args = record_args(argv);
+ let sources = record_sources(argv);
+ let mut settings = screenpipe_config::RecordingSettings {
+ capture_max_width: 2560,
+ ..Default::default()
+ };
+ args.apply_defaults_and_overrides(&mut settings, None, &sources);
+ assert_eq!(settings.capture_max_width, 1920);
+ assert!(sources.has_recording_override());
+ }
+
#[test]
fn test_fresh_hardware_defaults_survive_absent_cli_flags() {
use screenpipe_config::DeviceTier;
diff --git a/crates/screenpipe-engine/src/hd_recorder.rs b/crates/screenpipe-engine/src/hd_recorder.rs
index 6da750ba7..729aad78e 100644
--- a/crates/screenpipe-engine/src/hd_recorder.rs
+++ b/crates/screenpipe-engine/src/hd_recorder.rs
@@ -74,6 +74,9 @@ pub struct HdRecorderConfig {
pub ignore_incognito_windows: bool,
/// Let the incognito detector use browser-native APIs (macOS Automation).
pub enhanced_incognito_detection: bool,
+ /// Stored snapshot width from `videoQuality` (0 = native). Bounds the HD
+ /// stream; independent of the OCR capture width.
+ pub snapshot_max_width: u32,
}
/// Per-monitor HD recorder loop. Idles until an HD session is active, then
@@ -395,9 +398,12 @@ mod macos {
// starts the stream, so run it on a blocking thread.
let hd = {
let m = monitor.clone();
- tokio::task::spawn_blocking(move || m.start_hd_capture(fps, &excluded))
- .await
- .context("hd capture spawn_blocking join")??
+ let snapshot_max_width = config.snapshot_max_width;
+ tokio::task::spawn_blocking(move || {
+ m.start_hd_capture(fps, &excluded, snapshot_max_width)
+ })
+ .await
+ .context("hd capture spawn_blocking join")??
};
let actual_fps = hd.fps.max(1);
let (hd_w, hd_h) = (hd.width, hd.height);
diff --git a/crates/screenpipe-engine/src/recording_config.rs b/crates/screenpipe-engine/src/recording_config.rs
index ad083e961..e1797a843 100644
--- a/crates/screenpipe-engine/src/recording_config.rs
+++ b/crates/screenpipe-engine/src/recording_config.rs
@@ -18,6 +18,11 @@ use std::time::Duration;
use crate::vision_manager::VisionManagerConfig;
+/// Smallest non-native capture width. Narrower captures make OCR unreadable,
+/// and since stored snapshots are never upscaled they would also shrink every
+/// stored frame below the `low` preset (1280). Smaller values are raised.
+const MIN_CAPTURE_MAX_WIDTH: u32 = 1280;
+
/// Unified recording configuration used by both the CLI binary and the Tauri embedded server.
/// Replaces the former `EmbeddedServerConfig` and eliminates duplicate field mapping.
#[derive(Clone, Debug)]
@@ -162,6 +167,9 @@ pub struct RecordingConfig {
/// Video quality preset controlling JPEG quality during frame extraction.
/// Values: "low", "balanced", "high", "max". Default: "balanced".
pub video_quality: String,
+ /// Max width (px) of the macOS screen capture that OCR reads. 0 = native,
+ /// otherwise at least `MIN_CAPTURE_MAX_WIDTH`.
+ pub capture_max_width: u32,
// Misc
pub use_chinese_mirror: bool,
@@ -385,6 +393,10 @@ impl RecordingConfig {
openai_compatible_raw_audio: settings.openai_compatible_raw_audio,
user_name: settings.user_name.clone(),
video_quality: settings.video_quality.clone(),
+ capture_max_width: match settings.capture_max_width {
+ 0 => 0,
+ width => width.max(MIN_CAPTURE_MAX_WIDTH),
+ },
use_chinese_mirror: settings.use_chinese_mirror,
analytics_enabled: settings.analytics_enabled,
analytics_id: settings.analytics_id.clone(),
@@ -547,6 +559,7 @@ impl RecordingConfig {
pause_on_drm_content: self.pause_on_drm_content,
languages: self.languages.clone(),
video_quality: self.video_quality.clone(),
+ capture_max_width: self.capture_max_width,
disable_screenshots: self.disable_screenshots,
enable_semantic_context: self.enable_semantic_context,
semantic_context_mode: self.semantic_context_mode,
@@ -802,6 +815,7 @@ mod tests {
monitor_ids: vec!["MONITOR-1".to_string()],
use_all_monitors: false,
video_quality: "high".to_string(),
+ capture_max_width: 2560,
idle_capture_interval_ms: Some(2_000),
visual_check_interval_ms: Some(350),
visual_change_threshold: Some(0.18),
@@ -830,6 +844,7 @@ mod tests {
assert!(vision.enhanced_incognito_detection);
assert!(vision.pause_on_drm_content);
assert_eq!(vision.video_quality, "high");
+ assert_eq!(vision.capture_max_width, 2560);
assert!(!vision.disable_screenshots);
assert_eq!(vision.idle_capture_interval_ms, Some(2_000));
assert_eq!(vision.visual_check_interval_ms, Some(350));
@@ -839,6 +854,40 @@ mod tests {
assert_eq!(vision.capture_on_clipboard, Some(true));
}
+ /// OCR reads the captured frame, so no storage preset may shrink the
+ /// capture by default (#7393).
+ #[test]
+ fn capture_defaults_to_native_for_every_video_quality() {
+ for quality in ["low", "balanced", "high", "max"] {
+ let settings = screenpipe_config::RecordingSettings {
+ video_quality: quality.to_string(),
+ ..Default::default()
+ };
+ let vision = build(&settings).to_vision_manager_config(
+ String::new(),
+ std::sync::Arc::new(PipelineMetrics::new()),
+ );
+ assert_eq!(vision.capture_max_width, 0, "videoQuality={quality}");
+ }
+ }
+
+ /// A tiny cap would store 1-pixel-wide frames and lose the recording.
+ #[test]
+ fn capture_max_width_below_floor_is_raised() {
+ for (configured, effective) in [(0, 0), (1, 1280), (1279, 1280), (1280, 1280), (2560, 2560)]
+ {
+ let settings = screenpipe_config::RecordingSettings {
+ capture_max_width: configured,
+ ..Default::default()
+ };
+ assert_eq!(
+ build(&settings).capture_max_width,
+ effective,
+ "captureMaxWidth={configured}"
+ );
+ }
+ }
+
fn langs(items: &[&str]) -> Vec<String> {
items.iter().map(|s| s.to_string()).collect()
}
diff --git a/crates/screenpipe-engine/src/vision_manager/manager.rs b/crates/screenpipe-engine/src/vision_manager/manager.rs
index cac6dada5..f3f729ed6 100644
--- a/crates/screenpipe-engine/src/vision_manager/manager.rs
+++ b/crates/screenpipe-engine/src/vision_manager/manager.rs
@@ -59,6 +59,9 @@ pub struct VisionManagerConfig {
/// snapshot max width via `screenpipe_core::video::*`. Values: "low",
/// "balanced" (default), "high", "max".
pub video_quality: String,
+ /// Max width (px) of the macOS screen capture that OCR reads. 0 = native.
+ /// Independent of `video_quality`, which only sizes what is stored.
+ pub capture_max_width: u32,
/// Skip screenshot pixels/JPEG/OCR while keeping accessibility-tree capture.
pub disable_screenshots: bool,
/// Enable the bounded semantic projection worker. Off by default.
@@ -627,15 +630,13 @@ impl VisionManager {
max_snapshot_width,
));
- // Cap the macOS SCK capture stream to the same width as the snapshot
- // writer. The GPU downscales before replayd delivers the framebuffer,
- // saving WindowServer composite + readback cost without affecting
- // anything that wasn't going to be downsized in user space anyway.
- // Text extraction is primarily a11y-tree-driven (unchanged) and OCR
- // runs only as a fallback; both see the same image they'd see after
- // the snapshot-writer downscale.
+ // OCR reads the captured frame, not the stored JPEG, so the capture
+ // cap decides OCR legibility. Never set it from `max_snapshot_width`
+ // or `videoQuality`: capping capture at the stored width garbled
+ // small text on wide displays (#7393). The snapshot writer shrinks
+ // its own copy.
#[cfg(target_os = "macos")]
- screenpipe_screen::monitor::set_sck_capture_max_width(max_snapshot_width);
+ screenpipe_screen::monitor::set_sck_capture_max_width(self.config.capture_max_width);
// Create activity feed for this monitor
let activity_feed = ActivityFeed::new();
@@ -726,6 +727,7 @@ impl VisionManager {
included_urls: self.config.included_urls.clone(),
ignore_incognito_windows: self.config.ignore_incognito_windows,
enhanced_incognito_detection: self.config.enhanced_incognito_detection,
+ snapshot_max_width: max_snapshot_width,
};
let hd_handle = self
.vision_handle
@@ -1035,6 +1037,7 @@ mod tests {
pause_on_drm_content: false,
languages: vec![Language::English],
video_quality: "balanced".to_string(),
+ capture_max_width: 0,
disable_screenshots: false,
enable_semantic_context,
semantic_context_mode: SemanticContextMode::Memory,
diff --git a/crates/screenpipe-screen/src/monitor/macos.rs b/crates/screenpipe-screen/src/monitor/macos.rs
index 35a3600c0..0d88e22b7 100644
--- a/crates/screenpipe-screen/src/monitor/macos.rs
+++ b/crates/screenpipe-screen/src/monitor/macos.rs
@@ -64,22 +64,18 @@ static SCK_E2E_LOOKUP_HANG_ARMED: std::sync::atomic::AtomicBool =
static SCK_E2E_CAPTURE_HANG_INJECTED: std::sync::atomic::AtomicBool =
std::sync::atomic::AtomicBool::new(false);
-/// Optional cap on captured width for the macOS SCK stream. The GPU
-/// downscales to fit before `replayd` delivers the framebuffer, so
+/// Optional cap on captured width for the macOS SCK screenshot stream. The
+/// GPU downscales to fit before `replayd` delivers the framebuffer, so
/// WindowServer composites a smaller surface. `0` = no cap (capture at
-/// native, the previous default).
+/// native, the default).
///
-/// Set by the vision manager from the user's `video_quality` setting via
-/// `video_quality_to_max_snapshot_width` so the captured resolution
-/// matches the resolution the snapshot writer would downscale to anyway.
-/// Coupling these means screenpipe captures exactly the pixels it stores —
-/// no wasted GPU readback at native res just to throw it away in user
-/// space before JPEG encoding.
+/// This is the resolution OCR reads. Storage is shrunk separately by
+/// `SnapshotWriter`, so do not set this from storage settings such as
+/// `video_quality`: doing so garbled small text on wide displays (#7393).
static SCK_CAPTURE_MAX_WIDTH: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
-/// Set the SCK capture-width cap. `0` = no cap. Reads the user's
-/// `video_quality` setting via the snapshot-width mapping; called by the
-/// vision manager when capture starts and when settings change.
+/// Set the SCK capture-width cap. `0` = no cap. Called by the vision manager
+/// from the user's `captureMaxWidth` setting when capture starts.
pub fn set_sck_capture_max_width(max_width: u32) {
SCK_CAPTURE_MAX_WIDTH.store(max_width, std::sync::atomic::Ordering::Relaxed);
}
@@ -1045,8 +1041,8 @@ fn hd_scaled_dims(src_w: u32, src_h: u32, max_width: u32) -> (u32, u32) {
impl SafeMonitor {
/// Start a dedicated high-fps HD capture stream for this monitor at `fps`,
- /// honoring the screenshot resolution cap while independently bounding the
- /// live meeting stream to 1920px wide.
+ /// honoring the stored snapshot width (`snapshot_max_width`, 0 = native)
+ /// while independently bounding the live meeting stream to 1920px wide.
///
/// Returns a live [`HdCapture`]: drain `frames` for RGBA frames, drop
/// `stream` to stop. This opens a SECOND ScreenCaptureKit stream alongside
@@ -1054,11 +1050,16 @@ impl SafeMonitor {
/// path. `excluded_window_ids` are excluded at the OS level — ignored /
/// private windows never reach the recorder. Blocks briefly while the
/// stream starts; call from a blocking context.
- pub fn start_hd_capture(&self, fps: u32, excluded_window_ids: &[u32]) -> Result<HdCapture> {
+ pub fn start_hd_capture(
+ &self,
+ fps: u32,
+ excluded_window_ids: &[u32],
+ snapshot_max_width: u32,
+ ) -> Result<HdCapture> {
let (width, height) = hd_scaled_dims(
self.monitor_data.width,
self.monitor_data.height,
- hd_capture_max_width(sck_capture_max_width()),
+ hd_capture_max_width(snapshot_max_width),
);
let (stream, frames) =
sck_rs::start_hd_capture(self.monitor_id, width, height, fps, excluded_window_ids)
diff --git a/crates/screenpipe-screen/src/snapshot_writer.rs b/crates/screenpipe-screen/src/snapshot_writer.rs
index bc48062a1..4e5b9e6ca 100644
--- a/crates/screenpipe-screen/src/snapshot_writer.rs
+++ b/crates/screenpipe-screen/src/snapshot_writer.rs
@@ -177,6 +177,20 @@ mod tests {
assert_eq!(loaded.height(), 1080);
}
+ /// Capture can be wider than storage (#7393), so the writer itself must
+ /// cap the stored JPEG.
+ #[test]
+ fn test_write_downscales_to_max_width() {
+ let tmp = TempDir::new().unwrap();
+ let writer = SnapshotWriter::new(tmp.path(), 80, 1920);
+ let img = test_image(4000, 2000);
+
+ let path = writer.write(&img, Utc::now(), 0).unwrap();
+
+ let loaded = image::open(&path).unwrap();
+ assert_eq!((loaded.width(), loaded.height()), (1920, 960));
+ }
+
#[test]
fn test_write_creates_date_directory() {
let tmp = TempDir::new().unwrap();
diff --git a/docs/TESTING.md b/docs/TESTING.md
index 082d46c81..a43f24558 100644
--- a/docs/TESTING.md
+++ b/docs/TESTING.md
@@ -242,6 +242,7 @@ commits: `6dd5d98e`, `831ad258`
- [ ] **identical frames skipped** — check logs for hash match frequency on idle monitors. should be >80% skip rate.
- [ ] **ultrawide monitor (3440x1440+)** — OCR works correctly. no distortion in change detection. text at edges is captured.
- [ ] **4K monitor** — OCR works. frame comparison doesn't timeout or spike CPU.
+- [ ] **OCR reads full-size frames on wide displays (macOS)** — On a display wider than 1920 points with default settings (`balanced`), open a terminal with small text and search for a word from it: OCR finds it. Stored JPEGs stay at the `videoQuality` width (1920 for `balanced`), and meeting HD video width is unchanged. With `balanced`, `--capture-max-width 1920` restores the old capture size and cost. Capture width must never be derived from `videoQuality` (#7393).
- [ ] **high refresh rate (120Hz+)** — app respects its own FPS setting (0.5 default), not the display refresh rate.
- [ ] **very fast content changes** — scroll quickly through a document. OCR captures content, no crashes from buffer overflows.
- [ ] **corrupt pixel buffer** — sck-rs handles corrupt ScreenCaptureKit buffers gracefully (no SIGABRT). fixed in `831ad258`.