π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
fix(desktop): frame macOS menu clicks as the whole screen, named after the app in front
Luis Ortiz2e22bbc36ee5
diff --git a/apps/desktop/src/main/capture/focused-window.ts b/apps/desktop/src/main/capture/focused-window.tsindex 8e0bd11..482d98b 100644--- a/apps/desktop/src/main/capture/focused-window.ts+++ b/apps/desktop/src/main/capture/focused-window.ts@@ -35,7 +35,7 @@ function fromNative(found: ActiveWindow): FocusedWindowResult {window: {title: found.title ?? null,app: { name: found.appName, id: found.appPath },- bounds: toDip({ x: found.x, y: found.y, width: found.width, height: found.height }),+ bounds: found.onMenu ? null : toDip({ x: found.x, y: found.y, width: found.width, height: found.height }),},};}diff --git a/packages/capture-native/index.d.ts b/packages/capture-native/index.d.tsindex 803fa9b..c95ac6c 100644--- a/packages/capture-native/index.d.ts+++ b/packages/capture-native/index.d.ts@@ -19,6 +19,7 @@ export interface ActiveWindow {y: numberwidth: numberheight: number+ onMenu: boolean}export declare function clearDeadKey(): voiddiff --git a/packages/capture-native/src/lib.rs b/packages/capture-native/src/lib.rsindex abba167..45e3e3b 100644--- a/packages/capture-native/src/lib.rs+++ b/packages/capture-native/src/lib.rs@@ -57,6 +57,7 @@ pub struct ActiveWindow {pub y: f64,pub width: f64,pub height: f64,+ pub on_menu: bool,}#[napi(object)]diff --git a/packages/capture-native/src/mac.rs b/packages/capture-native/src/mac.rsindex ab0e411..5319a0d 100644--- a/packages/capture-native/src/mac.rs+++ b/packages/capture-native/src/mac.rs@@ -6,7 +6,10 @@ use std::sync::Mutex;use napi::Result;-use crate::macmap::{is_control, is_label, mac_keycode, named_key, printable, role, PROMOTE_DEPTH};+use crate::macmap::{+ is_control, is_label, mac_keycode, named_key, printable, role, window_for_click, LayeredWindow,+ PROMOTE_DEPTH,+};use crate::{ActiveWindow, ElementNode, ElementRect, UiElement};type CFTypeRef = *const c_void;@@ -447,6 +450,7 @@ pub fn clear_dead_key() {struct WindowInfo {pid: i32,+ layer: f64,owner: Option<String>,title: Option<String>,bounds: CGRect,@@ -466,7 +470,7 @@ fn number(dictionary: CFDictionaryRef, key: CFStringRef, kind: isize) -> Option<}}-fn windows() -> Vec<WindowInfo> {+fn windows(any_layer: bool) -> Vec<WindowInfo> {let Some(list) = Owned::new(unsafe { CGWindowListCopyWindowInfo(ON_SCREEN_ONLY | EXCLUDE_DESKTOP, 0) })else {return Vec::new();@@ -478,7 +482,7 @@ fn windows() -> Vec<WindowInfo> {let layer = number(entry, unsafe { kCGWindowLayer }, NUMBER_INT32)?;let alpha = number(entry, unsafe { kCGWindowAlpha }, NUMBER_FLOAT64).unwrap_or(1.0);let pid = number(entry, unsafe { kCGWindowOwnerPID }, NUMBER_INT32)? as i32;- if layer != 0.0 || alpha <= 0.0 || pid == own {+ if (!any_layer && layer != 0.0) || alpha <= 0.0 || pid == own {return None;}let mut bounds = CGRect::default();@@ -488,6 +492,7 @@ fn windows() -> Vec<WindowInfo> {}(bounds.size.width > 1.0 && bounds.size.height > 1.0).then(|| WindowInfo {pid,+ layer,owner: string_of(unsafe { CFDictionaryGetValue(entry, kCGWindowOwnerName) }),title: string_of(unsafe { CFDictionaryGetValue(entry, kCGWindowName) }),bounds,@@ -512,7 +517,7 @@ fn app_bundle(path: &str) -> String {.map_or_else(|| path.to_string(), |found| found.to_string_lossy().into_owned())}-fn framed(found: &WindowInfo) -> ActiveWindow {+fn framed(found: &WindowInfo, on_menu: bool) -> ActiveWindow {let path = process_path(found.pid).map(|path| app_bundle(&path));let app_name = found.owner@@ -532,6 +537,7 @@ fn framed(found: &WindowInfo) -> ActiveWindow {y: found.bounds.origin.y,width: found.bounds.size.width,height: found.bounds.size.height,+ on_menu,}}@@ -542,25 +548,31 @@ fn frontmost_pid() -> Option<i32> {}pub fn active_window() -> Option<ActiveWindow> {- let all = windows();+ let all = windows(false);let front = frontmost_pid();all.iter().find(|found| Some(found.pid) == front).or_else(|| all.first())- .map(framed)+ .map(|found| framed(found, false))}pub fn window_at(x: i32, y: i32) -> Option<ActiveWindow> {let (x, y) = (f64::from(x), f64::from(y));- windows()+ let all = windows(true);+ let layered: Vec<LayeredWindow> = all.iter()- .find(|found| {+ .map(|found| {let bounds = found.bounds;- x >= bounds.origin.x- && y >= bounds.origin.y- && x < bounds.origin.x + bounds.size.width- && y < bounds.origin.y + bounds.size.height+ LayeredWindow {+ pid: found.pid,+ layer: found.layer,+ under_point: x >= bounds.origin.x+ && y >= bounds.origin.y+ && x < bounds.origin.x + bounds.size.width+ && y < bounds.origin.y + bounds.size.height,+ }})- .map(framed)+ .collect();+ window_for_click(&layered, frontmost_pid()).map(|(index, on_menu)| framed(&all[index], on_menu))}diff --git a/packages/capture-native/src/macmap.rs b/packages/capture-native/src/macmap.rsindex 124287d..38347a5 100644--- a/packages/capture-native/src/macmap.rs+++ b/packages/capture-native/src/macmap.rs@@ -223,13 +223,42 @@ pub fn named_key(keycode: u32) -> Option<String> {.map(|(_, number)| format!("F{number}"))}+pub const MENU_BAR_LAYER: f64 = 24.0;++pub struct LayeredWindow {+ pub pid: i32,+ pub layer: f64,+ pub under_point: bool,+}++pub fn window_for_click(windows: &[LayeredWindow], front: Option<i32>) -> Option<(usize, bool)> {+ let hit = windows.iter().find(|found| found.under_point)?;+ if hit.layer == 0.0 {+ return windows+ .iter()+ .position(|found| found.under_point)+ .map(|index| (index, false));+ }+ let front = front?;+ if hit.pid != front && hit.layer != MENU_BAR_LAYER {+ return None;+ }+ windows+ .iter()+ .position(|found| found.layer == 0.0 && found.pid == front)+ .map(|index| (index, true))+}+pub fn printable(text: &str) -> Option<String> {(!text.is_empty() && text.chars().all(|found| !found.is_control())).then(|| text.to_string())}#[cfg(test)]mod tests {- use super::{hook_keycode, is_control, is_label, mac_keycode, named_key, printable, role};+ use super::{+ hook_keycode, is_control, is_label, mac_keycode, named_key, printable, role, window_for_click,+ LayeredWindow, MENU_BAR_LAYER,+ };#[test]fn mac_keys_come_back_as_hook_codes() {@@ -296,6 +325,37 @@ mod tests {assert_eq!(named_key(0x1e), None);}+ fn window(pid: i32, layer: f64, under_point: bool) -> LayeredWindow {+ LayeredWindow {+ pid,+ layer,+ under_point,+ }+ }++ #[test]+ fn menus_belong_to_the_app_in_front() {+ let menu_over_another_app = [+ window(7, 101.0, true),+ window(7, 0.0, false),+ window(9, 0.0, true),+ ];+ assert_eq!(window_for_click(&menu_over_another_app, Some(7)), Some((1, true)));+ let menu_over_its_own_app = [window(7, 101.0, true), window(7, 0.0, true)];+ assert_eq!(window_for_click(&menu_over_its_own_app, Some(7)), Some((1, true)));+ let menu_bar = [+ window(0, MENU_BAR_LAYER, true),+ window(9, 0.0, false),+ window(7, 0.0, false),+ ];+ assert_eq!(window_for_click(&menu_bar, Some(7)), Some((2, true)));+ let plain = [window(9, 0.0, true), window(7, 0.0, true)];+ assert_eq!(window_for_click(&plain, Some(7)), Some((0, false)));+ let dock = [window(3, 20.0, true), window(7, 0.0, true)];+ assert_eq!(window_for_click(&dock, Some(7)), None);+ assert_eq!(window_for_click(&menu_bar, None), None);+ }+#[test]fn control_characters_are_not_typing() {assert_eq!(printable("Γ©").as_deref(), Some("Γ©"));diff --git a/packages/capture-native/src/win.rs b/packages/capture-native/src/win.rsindex f3110e2..30299ad 100644--- a/packages/capture-native/src/win.rs+++ b/packages/capture-native/src/win.rs@@ -581,5 +581,6 @@ fn framed(front: HWND) -> Option<ActiveWindow> {y: f64::from(rect.top),width: f64::from(rect.right - rect.left),height: f64::from(rect.bottom - rect.top),+ on_menu: false,})}