Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

fix(desktop): frame macOS menu clicks as the whole screen, named after the app in front

Luis Ortiz2e22bbc36ee5
diff --git a/apps/desktop/src/main/capture/focused-window.ts b/apps/desktop/src/main/capture/focused-window.ts
index 8e0bd11..482d98b 100644
--- a/apps/desktop/src/main/capture/focused-window.ts
+++ b/apps/desktop/src/main/capture/focused-window.ts
@@ -35,7 +35,7 @@ function fromNative(found: ActiveWindow): FocusedWindowResult {
window: {
title: found.title ?? null,
app: { name: found.appName, id: found.appPath },
- bounds: toDip({ x: found.x, y: found.y, width: found.width, height: found.height }),
+ bounds: found.onMenu ? null : toDip({ x: found.x, y: found.y, width: found.width, height: found.height }),
},
};
}
diff --git a/packages/capture-native/index.d.ts b/packages/capture-native/index.d.ts
index 803fa9b..c95ac6c 100644
--- a/packages/capture-native/index.d.ts
+++ b/packages/capture-native/index.d.ts
@@ -19,6 +19,7 @@ export interface ActiveWindow {
y: number
width: number
height: number
+ onMenu: boolean
}
export declare function clearDeadKey(): void
diff --git a/packages/capture-native/src/lib.rs b/packages/capture-native/src/lib.rs
index abba167..45e3e3b 100644
--- a/packages/capture-native/src/lib.rs
+++ b/packages/capture-native/src/lib.rs
@@ -57,6 +57,7 @@ pub struct ActiveWindow {
pub y: f64,
pub width: f64,
pub height: f64,
+ pub on_menu: bool,
}
#[napi(object)]
diff --git a/packages/capture-native/src/mac.rs b/packages/capture-native/src/mac.rs
index ab0e411..5319a0d 100644
--- a/packages/capture-native/src/mac.rs
+++ b/packages/capture-native/src/mac.rs
@@ -6,7 +6,10 @@ use std::sync::Mutex;
use napi::Result;
-use crate::macmap::{is_control, is_label, mac_keycode, named_key, printable, role, PROMOTE_DEPTH};
+use crate::macmap::{
+ is_control, is_label, mac_keycode, named_key, printable, role, window_for_click, LayeredWindow,
+ PROMOTE_DEPTH,
+};
use crate::{ActiveWindow, ElementNode, ElementRect, UiElement};
type CFTypeRef = *const c_void;
@@ -447,6 +450,7 @@ pub fn clear_dead_key() {
struct WindowInfo {
pid: i32,
+ layer: f64,
owner: Option<String>,
title: Option<String>,
bounds: CGRect,
@@ -466,7 +470,7 @@ fn number(dictionary: CFDictionaryRef, key: CFStringRef, kind: isize) -> Option<
}
}
-fn windows() -> Vec<WindowInfo> {
+fn windows(any_layer: bool) -> Vec<WindowInfo> {
let Some(list) = Owned::new(unsafe { CGWindowListCopyWindowInfo(ON_SCREEN_ONLY | EXCLUDE_DESKTOP, 0) })
else {
return Vec::new();
@@ -478,7 +482,7 @@ fn windows() -> Vec<WindowInfo> {
let layer = number(entry, unsafe { kCGWindowLayer }, NUMBER_INT32)?;
let alpha = number(entry, unsafe { kCGWindowAlpha }, NUMBER_FLOAT64).unwrap_or(1.0);
let pid = number(entry, unsafe { kCGWindowOwnerPID }, NUMBER_INT32)? as i32;
- if layer != 0.0 || alpha <= 0.0 || pid == own {
+ if (!any_layer && layer != 0.0) || alpha <= 0.0 || pid == own {
return None;
}
let mut bounds = CGRect::default();
@@ -488,6 +492,7 @@ fn windows() -> Vec<WindowInfo> {
}
(bounds.size.width > 1.0 && bounds.size.height > 1.0).then(|| WindowInfo {
pid,
+ layer,
owner: string_of(unsafe { CFDictionaryGetValue(entry, kCGWindowOwnerName) }),
title: string_of(unsafe { CFDictionaryGetValue(entry, kCGWindowName) }),
bounds,
@@ -512,7 +517,7 @@ fn app_bundle(path: &str) -> String {
.map_or_else(|| path.to_string(), |found| found.to_string_lossy().into_owned())
}
-fn framed(found: &WindowInfo) -> ActiveWindow {
+fn framed(found: &WindowInfo, on_menu: bool) -> ActiveWindow {
let path = process_path(found.pid).map(|path| app_bundle(&path));
let app_name = found
.owner
@@ -532,6 +537,7 @@ fn framed(found: &WindowInfo) -> ActiveWindow {
y: found.bounds.origin.y,
width: found.bounds.size.width,
height: found.bounds.size.height,
+ on_menu,
}
}
@@ -542,25 +548,31 @@ fn frontmost_pid() -> Option<i32> {
}
pub fn active_window() -> Option<ActiveWindow> {
- let all = windows();
+ let all = windows(false);
let front = frontmost_pid();
all
.iter()
.find(|found| Some(found.pid) == front)
.or_else(|| all.first())
- .map(framed)
+ .map(|found| framed(found, false))
}
pub fn window_at(x: i32, y: i32) -> Option<ActiveWindow> {
let (x, y) = (f64::from(x), f64::from(y));
- windows()
+ let all = windows(true);
+ let layered: Vec<LayeredWindow> = all
.iter()
- .find(|found| {
+ .map(|found| {
let bounds = found.bounds;
- x >= bounds.origin.x
- && y >= bounds.origin.y
- && x < bounds.origin.x + bounds.size.width
- && y < bounds.origin.y + bounds.size.height
+ LayeredWindow {
+ pid: found.pid,
+ layer: found.layer,
+ under_point: x >= bounds.origin.x
+ && y >= bounds.origin.y
+ && x < bounds.origin.x + bounds.size.width
+ && y < bounds.origin.y + bounds.size.height,
+ }
})
- .map(framed)
+ .collect();
+ window_for_click(&layered, frontmost_pid()).map(|(index, on_menu)| framed(&all[index], on_menu))
}
diff --git a/packages/capture-native/src/macmap.rs b/packages/capture-native/src/macmap.rs
index 124287d..38347a5 100644
--- a/packages/capture-native/src/macmap.rs
+++ b/packages/capture-native/src/macmap.rs
@@ -223,13 +223,42 @@ pub fn named_key(keycode: u32) -> Option<String> {
.map(|(_, number)| format!("F{number}"))
}
+pub const MENU_BAR_LAYER: f64 = 24.0;
+
+pub struct LayeredWindow {
+ pub pid: i32,
+ pub layer: f64,
+ pub under_point: bool,
+}
+
+pub fn window_for_click(windows: &[LayeredWindow], front: Option<i32>) -> Option<(usize, bool)> {
+ let hit = windows.iter().find(|found| found.under_point)?;
+ if hit.layer == 0.0 {
+ return windows
+ .iter()
+ .position(|found| found.under_point)
+ .map(|index| (index, false));
+ }
+ let front = front?;
+ if hit.pid != front && hit.layer != MENU_BAR_LAYER {
+ return None;
+ }
+ windows
+ .iter()
+ .position(|found| found.layer == 0.0 && found.pid == front)
+ .map(|index| (index, true))
+}
+
pub fn printable(text: &str) -> Option<String> {
(!text.is_empty() && text.chars().all(|found| !found.is_control())).then(|| text.to_string())
}
#[cfg(test)]
mod tests {
- use super::{hook_keycode, is_control, is_label, mac_keycode, named_key, printable, role};
+ use super::{
+ hook_keycode, is_control, is_label, mac_keycode, named_key, printable, role, window_for_click,
+ LayeredWindow, MENU_BAR_LAYER,
+ };
#[test]
fn mac_keys_come_back_as_hook_codes() {
@@ -296,6 +325,37 @@ mod tests {
assert_eq!(named_key(0x1e), None);
}
+ fn window(pid: i32, layer: f64, under_point: bool) -> LayeredWindow {
+ LayeredWindow {
+ pid,
+ layer,
+ under_point,
+ }
+ }
+
+ #[test]
+ fn menus_belong_to_the_app_in_front() {
+ let menu_over_another_app = [
+ window(7, 101.0, true),
+ window(7, 0.0, false),
+ window(9, 0.0, true),
+ ];
+ assert_eq!(window_for_click(&menu_over_another_app, Some(7)), Some((1, true)));
+ let menu_over_its_own_app = [window(7, 101.0, true), window(7, 0.0, true)];
+ assert_eq!(window_for_click(&menu_over_its_own_app, Some(7)), Some((1, true)));
+ let menu_bar = [
+ window(0, MENU_BAR_LAYER, true),
+ window(9, 0.0, false),
+ window(7, 0.0, false),
+ ];
+ assert_eq!(window_for_click(&menu_bar, Some(7)), Some((2, true)));
+ let plain = [window(9, 0.0, true), window(7, 0.0, true)];
+ assert_eq!(window_for_click(&plain, Some(7)), Some((0, false)));
+ let dock = [window(3, 20.0, true), window(7, 0.0, true)];
+ assert_eq!(window_for_click(&dock, Some(7)), None);
+ assert_eq!(window_for_click(&menu_bar, None), None);
+ }
+
#[test]
fn control_characters_are_not_typing() {
assert_eq!(printable("Γ©").as_deref(), Some("Γ©"));
diff --git a/packages/capture-native/src/win.rs b/packages/capture-native/src/win.rs
index f3110e2..30299ad 100644
--- a/packages/capture-native/src/win.rs
+++ b/packages/capture-native/src/win.rs
@@ -581,5 +581,6 @@ fn framed(front: HWND) -> Option<ActiveWindow> {
y: f64::from(rect.top),
width: f64::from(rect.right - rect.left),
height: f64::from(rect.bottom - rect.top),
+ on_menu: false,
})
}