π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
use std::collections::HashSet;
use std::ffi::{c_char, c_void, CStr};
use std::path::Path;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::Mutex;
use napi::Result;
use crate::hit::{smallest_under, Bounds};
use crate::macmap::{
is_control, is_label, mac_keycode, named_key, printable, role, window_for_click, LayeredWindow,
PROMOTE_DEPTH,
};
use crate::{ActiveWindow, ElementNode, ElementRect, UiElement};
type CFTypeRef = *const c_void;
type CFStringRef = *const c_void;
type CFArrayRef = *const c_void;
type CFDictionaryRef = *const c_void;
type AXUIElementRef = *const c_void;
type AXError = i32;
type Boolean = u8;
const UTF8: u32 = 0x0800_0100;
const AX_POINT: u32 = 1;
const AX_SIZE: u32 = 2;
const NUMBER_INT32: isize = 3;
const NUMBER_FLOAT64: isize = 6;
const ON_SCREEN_ONLY: u32 = 1;
const EXCLUDE_DESKTOP: u32 = 16;
const AX_CALL_TIMEOUT: f32 = 1.0;
const MAX_ANCESTORS: usize = 4;
const MAX_CHILDREN: usize = 12;
const NARROW_DEPTH: usize = 4;
const NARROW_LIMIT: usize = 200;
#[repr(C)]
#[derive(Default, Clone, Copy)]
struct CGPoint {
x: f64,
y: f64,
}
#[repr(C)]
#[derive(Default, Clone, Copy)]
struct CGSize {
width: f64,
height: f64,
}
#[repr(C)]
#[derive(Default, Clone, Copy)]
struct CGRect {
origin: CGPoint,
size: CGSize,
}
#[link(name = "CoreFoundation", kind = "framework")]
extern "C" {
fn CFRelease(value: CFTypeRef);
fn CFRetain(value: CFTypeRef) -> CFTypeRef;
fn CFGetTypeID(value: CFTypeRef) -> usize;
fn CFStringGetTypeID() -> usize;
fn CFNumberGetTypeID() -> usize;
fn CFArrayGetTypeID() -> usize;
fn CFStringCreateWithBytes(
allocator: *const c_void,
bytes: *const u8,
length: isize,
encoding: u32,
external: Boolean,
) -> CFStringRef;
fn CFStringGetLength(string: CFStringRef) -> isize;
fn CFStringGetMaximumSizeForEncoding(length: isize, encoding: u32) -> isize;
fn CFStringGetCString(string: CFStringRef, buffer: *mut c_char, size: isize, encoding: u32) -> Boolean;
fn CFArrayGetCount(array: CFArrayRef) -> isize;
fn CFArrayGetValueAtIndex(array: CFArrayRef, index: isize) -> *const c_void;
fn CFDictionaryGetValue(dictionary: CFDictionaryRef, key: *const c_void) -> *const c_void;
fn CFNumberGetValue(number: CFTypeRef, kind: isize, value: *mut c_void) -> Boolean;
fn CFDataGetBytePtr(data: CFTypeRef) -> *const u8;
static kCFBooleanTrue: CFTypeRef;
static kCFBooleanFalse: CFTypeRef;
}
#[link(name = "ApplicationServices", kind = "framework")]
extern "C" {
fn AXUIElementCreateSystemWide() -> AXUIElementRef;
fn AXUIElementCreateApplication(pid: i32) -> AXUIElementRef;
fn AXUIElementCopyElementAtPosition(
application: AXUIElementRef,
x: f32,
y: f32,
element: *mut AXUIElementRef,
) -> AXError;
fn AXUIElementCopyAttributeValue(
element: AXUIElementRef,
attribute: CFStringRef,
value: *mut CFTypeRef,
) -> AXError;
fn AXUIElementSetAttributeValue(
element: AXUIElementRef,
attribute: CFStringRef,
value: CFTypeRef,
) -> AXError;
fn AXUIElementSetMessagingTimeout(element: AXUIElementRef, seconds: f32) -> AXError;
fn AXUIElementGetPid(element: AXUIElementRef, pid: *mut i32) -> AXError;
fn AXValueGetValue(value: CFTypeRef, kind: u32, out: *mut c_void) -> Boolean;
}
#[link(name = "CoreGraphics", kind = "framework")]
extern "C" {
fn CGWindowListCopyWindowInfo(option: u32, relative_to: u32) -> CFArrayRef;
fn CGRectMakeWithDictionaryRepresentation(dictionary: CFDictionaryRef, rect: *mut CGRect) -> bool;
fn CGEventSourceFlagsState(state: i32) -> u64;
static kCGWindowLayer: CFStringRef;
static kCGWindowBounds: CFStringRef;
static kCGWindowOwnerPID: CFStringRef;
static kCGWindowOwnerName: CFStringRef;
static kCGWindowName: CFStringRef;
static kCGWindowAlpha: CFStringRef;
}
#[link(name = "Carbon", kind = "framework")]
extern "C" {
fn TISCopyCurrentKeyboardLayoutInputSource() -> CFTypeRef;
fn TISCopyCurrentASCIICapableKeyboardLayoutInputSource() -> CFTypeRef;
fn TISGetInputSourceProperty(source: CFTypeRef, key: CFStringRef) -> CFTypeRef;
fn LMGetKbdType() -> u8;
fn UCKeyTranslate(
layout: *const u8,
keycode: u16,
action: u16,
modifiers: u32,
keyboard: u32,
options: u32,
dead_key: *mut u32,
max: usize,
length: *mut usize,
text: *mut u16,
) -> i32;
static kTISPropertyUnicodeKeyLayoutData: CFStringRef;
}
extern "C" {
fn proc_pidpath(pid: i32, buffer: *mut c_void, size: u32) -> i32;
}
struct Owned(CFTypeRef);
impl Owned {
fn new(value: CFTypeRef) -> Option<Self> {
(!value.is_null()).then_some(Self(value))
}
}
impl Drop for Owned {
fn drop(&mut self) {
unsafe { CFRelease(self.0) };
}
}
fn cf_string(text: &str) -> Owned {
Owned(unsafe { CFStringCreateWithBytes(std::ptr::null(), text.as_ptr(), text.len() as isize, UTF8, 0) })
}
fn clean(value: String) -> Option<String> {
let trimmed = value.trim();
(!trimmed.is_empty()).then(|| trimmed.to_string())
}
fn string_of(value: CFTypeRef) -> Option<String> {
if value.is_null() || unsafe { CFGetTypeID(value) != CFStringGetTypeID() } {
return None;
}
let length = unsafe { CFStringGetLength(value) };
let size = unsafe { CFStringGetMaximumSizeForEncoding(length, UTF8) } + 1;
let mut buffer = vec![0 as c_char; size as usize];
if unsafe { CFStringGetCString(value, buffer.as_mut_ptr(), size, UTF8) } == 0 {
return None;
}
let text = unsafe { CStr::from_ptr(buffer.as_ptr()) };
clean(text.to_string_lossy().into_owned())
}
fn attribute(element: AXUIElementRef, name: &str) -> Option<Owned> {
let key = cf_string(name);
let mut value: CFTypeRef = std::ptr::null();
let read = unsafe { AXUIElementCopyAttributeValue(element, key.0, &mut value) };
if read == 0 {
Owned::new(value)
} else {
None
}
}
fn text(element: AXUIElementRef, name: &str) -> Option<String> {
attribute(element, name).and_then(|value| string_of(value.0))
}
fn ax_value<T: Default>(element: AXUIElementRef, name: &str, kind: u32) -> Option<T> {
let value = attribute(element, name)?;
let mut out = T::default();
(unsafe { AXValueGetValue(value.0, kind, (&mut out as *mut T).cast()) } != 0).then_some(out)
}
fn rect_of(element: AXUIElementRef) -> Option<ElementRect> {
let origin: CGPoint = ax_value(element, "AXPosition", AX_POINT)?;
let size: CGSize = ax_value(element, "AXSize", AX_SIZE)?;
(size.width > 0.0 && size.height > 0.0).then_some(ElementRect {
x: origin.x,
y: origin.y,
width: size.width,
height: size.height,
})
}
fn role_of(element: AXUIElementRef) -> Option<String> {
role(
text(element, "AXRole").as_deref(),
text(element, "AXSubrole").as_deref(),
)
}
fn secure(element: AXUIElementRef) -> bool {
text(element, "AXSubrole").as_deref() == Some("AXSecureTextField")
}
fn secure_length(element: AXUIElementRef) -> Option<u32> {
let counted = attribute(element, "AXNumberOfCharacters")
.filter(|found| unsafe { CFGetTypeID(found.0) == CFNumberGetTypeID() })
.and_then(|found| {
let mut out = 0i32;
(unsafe { CFNumberGetValue(found.0, NUMBER_INT32, (&mut out as *mut i32).cast()) } != 0)
.then_some(out.max(0) as u32)
});
counted.or_else(|| text(element, "AXValue").map(|value| value.chars().count() as u32))
}
fn label_of(element: AXUIElementRef) -> Option<String> {
text(element, "AXTitle")
.or_else(|| text(element, "AXDescription"))
.or_else(|| {
let title = attribute(element, "AXTitleUIElement")?;
text(title.0, "AXValue").or_else(|| text(title.0, "AXTitle"))
})
}
fn describe(element: AXUIElementRef) -> UiElement {
let is_password = secure(element);
UiElement {
role: role_of(element),
name: label_of(element),
automation_id: text(element, "AXIdentifier"),
value: if is_password {
None
} else {
text(element, "AXValue")
},
help_text: text(element, "AXHelp").or_else(|| text(element, "AXPlaceholderValue")),
is_password,
value_length: if is_password { secure_length(element) } else { None },
rect: rect_of(element),
ancestors: Vec::new(),
children: Vec::new(),
}
}
fn node(element: AXUIElementRef) -> ElementNode {
ElementNode {
role: role_of(element),
name: label_of(element),
}
}
fn parent(element: AXUIElementRef) -> Option<Owned> {
attribute(element, "AXParent")
}
fn ancestors(element: AXUIElementRef) -> Vec<ElementNode> {
let mut out = Vec::new();
let mut current = parent(element);
while let Some(found) = current {
if out.len() == MAX_ANCESTORS {
break;
}
out.push(node(found.0));
current = parent(found.0);
}
out
}
fn children(element: AXUIElementRef) -> Vec<ElementNode> {
let Some(list) = attribute(element, "AXChildren") else {
return Vec::new();
};
if unsafe { CFGetTypeID(list.0) != CFArrayGetTypeID() } {
return Vec::new();
}
let count = unsafe { CFArrayGetCount(list.0) }.min(MAX_CHILDREN as isize);
(0..count)
.map(|index| node(unsafe { CFArrayGetValueAtIndex(list.0, index) }))
.collect()
}
fn promoted(hit: Owned) -> Owned {
if !is_label(role_of(hit.0).as_deref()) {
return hit;
}
let mut current = parent(hit.0);
for _ in 0..PROMOTE_DEPTH {
let Some(found) = current else { break };
if is_control(role_of(found.0).as_deref()) {
return found;
}
current = parent(found.0);
}
hit
}
fn child_list(element: AXUIElementRef) -> Option<Owned> {
attribute(element, "AXChildren").filter(|list| unsafe { CFGetTypeID(list.0) == CFArrayGetTypeID() })
}
fn is_cover(element: AXUIElementRef) -> bool {
role_of(element).as_deref() == Some("group")
&& label_of(element).is_none()
&& child_list(element).is_none_or(|list| unsafe { CFArrayGetCount(list.0) } == 0)
}
fn bounds(rect: ElementRect) -> Bounds {
(
rect.x as i32,
rect.y as i32,
(rect.x + rect.width) as i32,
(rect.y + rect.height) as i32,
)
}
fn narrowed(hit: Owned, x: i32, y: i32) -> Owned {
if !is_cover(hit.0) {
return hit;
}
let Some(holder) = parent(hit.0) else { return hit };
let mut lists = Vec::new();
let mut elements = Vec::new();
let mut boxes = Vec::new();
let mut level = vec![holder.0];
for _ in 0..NARROW_DEPTH {
let mut next = Vec::new();
for element in level {
let Some(list) = child_list(element) else { continue };
for index in 0..unsafe { CFArrayGetCount(list.0) } {
let child = unsafe { CFArrayGetValueAtIndex(list.0, index) };
if let Some(rect) = rect_of(child) {
elements.push(child);
boxes.push(bounds(rect));
}
next.push(child);
}
lists.push(list);
}
if boxes.len() >= NARROW_LIMIT {
break;
}
level = next;
}
smallest_under(&boxes, x, y)
.and_then(|index| Owned::new(unsafe { CFRetain(elements[index]) }))
.unwrap_or(hit)
}
static OPENED: Mutex<Option<HashSet<i32>>> = Mutex::new(None);
static ENHANCED: Mutex<Vec<i32>> = Mutex::new(Vec::new());
const ENHANCED_UI: &str = "AXEnhancedUserInterface";
fn pid_of(element: AXUIElementRef) -> Option<i32> {
let mut pid = 0;
(unsafe { AXUIElementGetPid(element, &mut pid) } == 0).then_some(pid)
}
fn ours(element: AXUIElementRef) -> bool {
pid_of(element) == Some(std::process::id() as i32)
}
fn open_web_content(element: AXUIElementRef) -> bool {
let Some(pid) = pid_of(element).filter(|_| !ours(element)) else {
return false;
};
let mut opened = OPENED.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
if !opened.get_or_insert_with(HashSet::new).insert(pid) {
return false;
}
let Some(app) = Owned::new(unsafe { AXUIElementCreateApplication(pid) }) else {
return false;
};
let manual = cf_string("AXManualAccessibility");
let opened = unsafe { AXUIElementSetAttributeValue(app.0, manual.0, kCFBooleanTrue) == 0 };
let already = attribute(app.0, ENHANCED_UI).is_some_and(|value| value.0 == unsafe { kCFBooleanTrue });
let enhanced = cf_string(ENHANCED_UI);
if !already && unsafe { AXUIElementSetAttributeValue(app.0, enhanced.0, kCFBooleanTrue) == 0 } {
ENHANCED
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.push(pid);
return true;
}
opened
}
pub fn release_web_content() {
OPENED
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.take();
let pids = std::mem::take(&mut *ENHANCED.lock().unwrap_or_else(|poisoned| poisoned.into_inner()));
let key = cf_string(ENHANCED_UI);
for pid in pids {
if let Some(app) = Owned::new(unsafe { AXUIElementCreateApplication(pid) }) {
unsafe { AXUIElementSetAttributeValue(app.0, key.0, kCFBooleanFalse) };
}
}
}
fn system() -> Option<Owned> {
let wide = Owned::new(unsafe { AXUIElementCreateSystemWide() })?;
unsafe { AXUIElementSetMessagingTimeout(wide.0, AX_CALL_TIMEOUT) };
Some(wide)
}
fn hit_test(wide: &Owned, x: i32, y: i32) -> Option<Owned> {
let mut found: AXUIElementRef = std::ptr::null();
let read = unsafe { AXUIElementCopyElementAtPosition(wide.0, x as f32, y as f32, &mut found) };
if read == 0 {
Owned::new(found)
} else {
None
}
}
pub fn element_at_point(x: i32, y: i32) -> Result<Option<UiElement>> {
let Some(wide) = system() else { return Ok(None) };
let Some(mut hit) = hit_test(&wide, x, y).filter(|found| !ours(found.0)) else {
return Ok(None);
};
if open_web_content(hit.0) {
if let Some(again) = hit_test(&wide, x, y) {
hit = again;
}
}
let found = promoted(narrowed(hit, x, y));
let mut element = describe(found.0);
if element.name.is_none() {
element.ancestors = ancestors(found.0);
element.children = children(found.0);
}
Ok(Some(element))
}
pub fn focused_element() -> Result<Option<UiElement>> {
let Some(wide) = system() else { return Ok(None) };
Ok(attribute(wide.0, "AXFocusedUIElement").map(|found| describe(found.0)))
}
static DEAD_KEY: AtomicU32 = AtomicU32::new(0);
const SHIFT: u32 = 1 << 9;
const CAPS: u32 = 1 << 10;
const OPTION: u32 = 1 << 11;
const CONTROL: u32 = 1 << 12;
const NO_DEAD_KEYS: u32 = 1;
const CAPS_FLAG: u64 = 0x0001_0000;
const COMBINED_SESSION: i32 = 0;
fn translate(keycode: u16, modifiers: u32, options: u32, dead_key: &mut u32) -> Option<String> {
let source = Owned::new(unsafe { TISCopyCurrentKeyboardLayoutInputSource() })?;
let mut data = unsafe { TISGetInputSourceProperty(source.0, kTISPropertyUnicodeKeyLayoutData) };
let _fallback;
if data.is_null() {
let ascii = Owned::new(unsafe { TISCopyCurrentASCIICapableKeyboardLayoutInputSource() })?;
data = unsafe { TISGetInputSourceProperty(ascii.0, kTISPropertyUnicodeKeyLayoutData) };
_fallback = ascii;
}
if data.is_null() {
return None;
}
let layout = unsafe { CFDataGetBytePtr(data) };
let mut buffer = [0u16; 8];
let mut length = 0usize;
let status = unsafe {
UCKeyTranslate(
layout,
keycode,
0,
(modifiers >> 8) & 0xff,
u32::from(LMGetKbdType()),
options,
dead_key,
buffer.len(),
&mut length,
buffer.as_mut_ptr(),
)
};
if status != 0 || length == 0 {
return None;
}
String::from_utf16(&buffer[..length]).ok()
}
pub fn key_label(keycode: u32) -> Option<String> {
if let Some(name) = named_key(keycode) {
return Some(name);
}
let mac = mac_keycode(keycode)?;
[0, SHIFT].iter().find_map(|&modifiers| {
let mut ignored = 0;
translate(mac, modifiers, NO_DEAD_KEYS, &mut ignored)
.filter(|found| found.chars().count() == 1 && found.chars().all(|found| found.is_ascii_alphanumeric()))
.map(|found| found.to_ascii_uppercase())
})
}
pub fn resolve_key(keycode: u32, shift: bool, ctrl: bool, alt: bool) -> Option<String> {
let mac = mac_keycode(keycode)?;
let mut modifiers = 0;
if shift {
modifiers |= SHIFT;
}
if alt {
modifiers |= OPTION;
}
if ctrl {
modifiers |= CONTROL;
}
if unsafe { CGEventSourceFlagsState(COMBINED_SESSION) } & CAPS_FLAG != 0 {
modifiers |= CAPS;
}
let mut dead_key = DEAD_KEY.load(Ordering::Relaxed);
let found = translate(mac, modifiers, 0, &mut dead_key);
DEAD_KEY.store(dead_key, Ordering::Relaxed);
found.and_then(|found| printable(&found))
}
pub fn clear_dead_key() {
DEAD_KEY.store(0, Ordering::Relaxed);
}
struct WindowInfo {
pid: i32,
layer: f64,
owner: Option<String>,
title: Option<String>,
bounds: CGRect,
}
fn number(dictionary: CFDictionaryRef, key: CFStringRef, kind: isize) -> Option<f64> {
let value = unsafe { CFDictionaryGetValue(dictionary, key) };
if value.is_null() {
return None;
}
if kind == NUMBER_INT32 {
let mut out = 0i32;
(unsafe { CFNumberGetValue(value, kind, (&mut out as *mut i32).cast()) } != 0).then_some(f64::from(out))
} else {
let mut out = 0f64;
(unsafe { CFNumberGetValue(value, kind, (&mut out as *mut f64).cast()) } != 0).then_some(out)
}
}
fn windows(any_layer: bool) -> Vec<WindowInfo> {
let Some(list) = Owned::new(unsafe { CGWindowListCopyWindowInfo(ON_SCREEN_ONLY | EXCLUDE_DESKTOP, 0) })
else {
return Vec::new();
};
let own = std::process::id() as i32;
(0..unsafe { CFArrayGetCount(list.0) })
.filter_map(|index| {
let entry = unsafe { CFArrayGetValueAtIndex(list.0, index) };
let layer = number(entry, unsafe { kCGWindowLayer }, NUMBER_INT32)?;
let alpha = number(entry, unsafe { kCGWindowAlpha }, NUMBER_FLOAT64).unwrap_or(1.0);
let pid = number(entry, unsafe { kCGWindowOwnerPID }, NUMBER_INT32)? as i32;
if (!any_layer && layer != 0.0) || alpha <= 0.0 || pid == own {
return None;
}
let mut bounds = CGRect::default();
let dictionary = unsafe { CFDictionaryGetValue(entry, kCGWindowBounds) };
if dictionary.is_null() || !unsafe { CGRectMakeWithDictionaryRepresentation(dictionary, &mut bounds) } {
return None;
}
(bounds.size.width > 1.0 && bounds.size.height > 1.0).then(|| WindowInfo {
pid,
layer,
owner: string_of(unsafe { CFDictionaryGetValue(entry, kCGWindowOwnerName) }),
title: string_of(unsafe { CFDictionaryGetValue(entry, kCGWindowName) }),
bounds,
})
})
.collect()
}
fn process_path(pid: i32) -> Option<String> {
let mut buffer = [0u8; 4096];
let length = unsafe { proc_pidpath(pid, buffer.as_mut_ptr().cast(), buffer.len() as u32) };
if length <= 0 {
return None;
}
clean(String::from_utf8_lossy(&buffer[..length as usize]).into_owned())
}
fn app_bundle(path: &str) -> String {
Path::new(path)
.ancestors()
.find(|found| found.extension().is_some_and(|extension| extension == "app"))
.map_or_else(|| path.to_string(), |found| found.to_string_lossy().into_owned())
}
fn framed(found: &WindowInfo, on_menu: bool) -> ActiveWindow {
let path = process_path(found.pid).map(|path| app_bundle(&path));
let app_name = found
.owner
.clone()
.or_else(|| {
path
.as_deref()
.and_then(|found| Path::new(found).file_stem())
.map(|stem| stem.to_string_lossy().into_owned())
})
.unwrap_or_default();
ActiveWindow {
title: found.title.clone(),
app_name,
app_path: path,
x: found.bounds.origin.x,
y: found.bounds.origin.y,
width: found.bounds.size.width,
height: found.bounds.size.height,
on_menu,
}
}
fn frontmost_pid() -> Option<i32> {
let wide = system()?;
let app = attribute(wide.0, "AXFocusedApplication")?;
pid_of(app.0)
}
pub fn active_window() -> Option<ActiveWindow> {
let all = windows(false);
let front = frontmost_pid();
all
.iter()
.find(|found| Some(found.pid) == front)
.or_else(|| all.first())
.map(|found| framed(found, false))
}
pub fn window_at(x: i32, y: i32) -> Option<ActiveWindow> {
let (x, y) = (f64::from(x), f64::from(y));
let all = windows(true);
let layered: Vec<LayeredWindow> = all
.iter()
.map(|found| {
let bounds = found.bounds;
LayeredWindow {
pid: found.pid,
layer: found.layer,
under_point: x >= bounds.origin.x
&& y >= bounds.origin.y
&& x < bounds.origin.x + bounds.size.width
&& y < bounds.origin.y + bounds.size.height,
}
})
.collect();
window_for_click(&layered, frontmost_pid()).map(|(index, on_menu)| framed(&all[index], on_menu))
}