π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
use std::cell::RefCell;
use std::ffi::c_void;
use std::path::Path;
use napi::Result;
use windows::core::{Interface, BOOL, BSTR, PCWSTR, PWSTR};
use windows::Win32::Foundation::{CloseHandle, HWND, LPARAM, POINT, RECT};
use windows::Win32::Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_CLOAKED, DWMWA_EXTENDED_FRAME_BOUNDS};
use windows::Win32::Graphics::Gdi::{
GetMonitorInfoW, MonitorFromWindow, MONITORINFO, MONITOR_DEFAULTTONEAREST,
};
use windows::Win32::Storage::FileSystem::{GetFileVersionInfoSizeW, GetFileVersionInfoW, VerQueryValueW};
use windows::Win32::System::Com::{
CoCreateInstance, CoInitializeEx, CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED,
};
use windows::Win32::System::Threading::{
OpenProcess, QueryFullProcessImageNameW, PROCESS_NAME_WIN32, PROCESS_QUERY_LIMITED_INFORMATION,
};
use windows::Win32::UI::Accessibility::*;
use windows::Win32::UI::Input::KeyboardAndMouse::{
GetKeyState, GetKeyboardLayout, MapVirtualKeyExW, ToUnicodeEx, HKL, MAPVK_VK_TO_VSC, MAPVK_VSC_TO_VK_EX,
VK_BACK, VK_CAPITAL, VK_CONTROL, VK_DELETE, VK_DOWN, VK_END, VK_ESCAPE, VK_F1, VK_F24, VK_HOME, VK_INSERT,
VK_LEFT, VK_MENU, VK_NEXT, VK_PRIOR, VK_RETURN, VK_RIGHT, VK_SHIFT, VK_SPACE, VK_TAB, VK_UP,
};
use windows::Win32::UI::WindowsAndMessaging::{
EnumWindows, GetAncestor, GetClassNameW, GetForegroundWindow, GetWindow, GetWindowLongPtrW, GetWindowRect,
GetWindowTextW, GetWindowThreadProcessId, IsWindowVisible, WindowFromPoint, GA_ROOT, GWL_EXSTYLE,
GWL_STYLE, GW_OWNER, WS_CAPTION, WS_CHILD, WS_EX_NOACTIVATE, WS_EX_TOOLWINDOW, WS_POPUP, WS_SYSMENU,
};
use crate::hit::smallest_under;
use crate::window::{fills, is_popup, names_its_surface, Edges, Traits};
use crate::{ActiveWindow, ElementNode, ElementRect, UiElement};
const ROLES: &[(UIA_CONTROLTYPE_ID, &str)] = &[
(UIA_ButtonControlTypeId, "button"),
(UIA_CalendarControlTypeId, "calendar"),
(UIA_CheckBoxControlTypeId, "checkbox"),
(UIA_ComboBoxControlTypeId, "combobox"),
(UIA_EditControlTypeId, "textbox"),
(UIA_HyperlinkControlTypeId, "link"),
(UIA_ImageControlTypeId, "img"),
(UIA_ListItemControlTypeId, "listitem"),
(UIA_ListControlTypeId, "list"),
(UIA_MenuControlTypeId, "menu"),
(UIA_MenuBarControlTypeId, "menubar"),
(UIA_MenuItemControlTypeId, "menuitem"),
(UIA_ProgressBarControlTypeId, "progressbar"),
(UIA_RadioButtonControlTypeId, "radio"),
(UIA_ScrollBarControlTypeId, "scrollbar"),
(UIA_SliderControlTypeId, "slider"),
(UIA_SpinnerControlTypeId, "spinbutton"),
(UIA_StatusBarControlTypeId, "status"),
(UIA_TabControlTypeId, "tablist"),
(UIA_TabItemControlTypeId, "tab"),
(UIA_TextControlTypeId, "text"),
(UIA_ToolBarControlTypeId, "toolbar"),
(UIA_ToolTipControlTypeId, "tooltip"),
(UIA_TreeControlTypeId, "tree"),
(UIA_TreeItemControlTypeId, "treeitem"),
(UIA_GroupControlTypeId, "group"),
(UIA_ThumbControlTypeId, "thumb"),
(UIA_DataGridControlTypeId, "grid"),
(UIA_DataItemControlTypeId, "row"),
(UIA_DocumentControlTypeId, "document"),
(UIA_SplitButtonControlTypeId, "button"),
(UIA_WindowControlTypeId, "window"),
(UIA_PaneControlTypeId, "pane"),
(UIA_HeaderControlTypeId, "rowgroup"),
(UIA_HeaderItemControlTypeId, "columnheader"),
(UIA_TableControlTypeId, "table"),
(UIA_TitleBarControlTypeId, "banner"),
(UIA_SeparatorControlTypeId, "separator"),
(UIA_AppBarControlTypeId, "toolbar"),
];
thread_local! {
static AUTOMATION: RefCell<Option<IUIAutomation>> = const { RefCell::new(None) };
}
fn automation() -> windows::core::Result<IUIAutomation> {
AUTOMATION.with(|cell| {
if let Some(found) = cell.borrow().as_ref() {
return Ok(found.clone());
}
unsafe {
let _ = CoInitializeEx(None, COINIT_MULTITHREADED);
}
let created: IUIAutomation = unsafe { CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER)? };
*cell.borrow_mut() = Some(created.clone());
Ok(created)
})
}
fn clean(value: String) -> Option<String> {
let trimmed = value.trim();
if trimmed.is_empty() {
None
} else {
Some(trimmed.to_string())
}
}
fn text(value: windows::core::Result<BSTR>) -> Option<String> {
clean(value.ok()?.to_string())
}
fn role(id: UIA_CONTROLTYPE_ID) -> Option<String> {
ROLES
.iter()
.find(|(key, _)| key.0 == id.0)
.map(|(_, name)| (*name).to_string())
}
fn value_of(element: &IUIAutomationElement) -> Option<String> {
let pattern = unsafe { element.GetCurrentPattern(UIA_ValuePatternId) }.ok()?;
let value = pattern.cast::<IUIAutomationValuePattern>().ok()?;
text(unsafe { value.CurrentValue() })
}
fn rect_of(element: &IUIAutomationElement) -> Option<ElementRect> {
let found = unsafe { element.CurrentBoundingRectangle() }.ok()?;
let width = f64::from(found.right - found.left);
let height = f64::from(found.bottom - found.top);
if width <= 0.0 || height <= 0.0 {
return None;
}
Some(ElementRect {
x: f64::from(found.left),
y: f64::from(found.top),
width,
height,
})
}
fn shown_name(element: &IUIAutomationElement, control: UIA_CONTROLTYPE_ID) -> Option<String> {
let name = text(unsafe { element.CurrentName() })?;
let class = text(unsafe { element.CurrentClassName() }).unwrap_or_default();
let container = [
UIA_PaneControlTypeId,
UIA_DocumentControlTypeId,
UIA_WindowControlTypeId,
]
.contains(&control);
let hidden_caption = unsafe { element.CurrentNativeWindowHandle() }
.ok()
.filter(|window| !window.is_invalid() && style(*window) & WS_CHILD.0 != 0)
.and_then(|window| text_of(|buffer| unsafe { GetWindowTextW(window, buffer) }));
(!names_its_surface(&name, &class, container, hidden_caption.as_deref())).then_some(name)
}
fn describe(found: &IUIAutomationElement) -> UiElement {
let control = unsafe { found.CurrentControlType() }.unwrap_or_default();
UiElement {
role: role(control),
name: shown_name(found, control),
automation_id: text(unsafe { found.CurrentAutomationId() }),
value: value_of(found),
help_text: text(unsafe { found.CurrentHelpText() }),
is_password: unsafe { found.CurrentIsPassword() }
.map(|flag| flag.as_bool())
.unwrap_or(false),
value_length: None,
rect: rect_of(found),
ancestors: Vec::new(),
children: Vec::new(),
}
}
const MAX_ANCESTORS: usize = 4;
const TAB_DRAG_LAYER: &str = "TabDragContextImpl";
const MAX_CHILDREN: usize = 12;
fn node(element: &IUIAutomationElement) -> ElementNode {
let control = unsafe { element.CurrentControlType() }.unwrap_or_default();
ElementNode {
role: role(control),
name: shown_name(element, control),
}
}
fn ancestors(walker: &IUIAutomationTreeWalker, element: &IUIAutomationElement) -> Vec<ElementNode> {
let mut out = Vec::new();
let mut current = element.clone();
while out.len() < MAX_ANCESTORS {
let Ok(parent) = (unsafe { walker.GetParentElement(¤t) }) else {
break;
};
out.push(node(&parent));
current = parent;
}
out
}
fn children(walker: &IUIAutomationTreeWalker, element: &IUIAutomationElement) -> Vec<ElementNode> {
let mut out = Vec::new();
let mut next = unsafe { walker.GetFirstChildElement(element) };
while let Ok(child) = next {
if out.len() == MAX_CHILDREN {
break;
}
out.push(node(&child));
next = unsafe { walker.GetNextSiblingElement(&child) };
}
out
}
fn narrowest(
uia: &IUIAutomation,
walker: &IUIAutomationTreeWalker,
hit: &IUIAutomationElement,
point: POINT,
) -> Option<IUIAutomationElement> {
if unsafe { hit.CurrentControlType() }.ok()? == UIA_EditControlTypeId {
return None;
}
unsafe { walker.GetFirstChildElement(hit) }.ok()?;
let request = unsafe { uia.CreateCacheRequest() }.ok()?;
unsafe { request.AddProperty(UIA_BoundingRectanglePropertyId) }.ok()?;
unsafe { request.AddProperty(UIA_ClassNamePropertyId) }.ok()?;
let everything = unsafe { uia.CreateTrueCondition() }.ok()?;
let found = unsafe { hit.FindAllBuildCache(TreeScope_Subtree, &everything, &request) }.ok()?;
let elements: Vec<IUIAutomationElement> = (0..unsafe { found.Length() }.ok()?)
.filter_map(|index| unsafe { found.GetElement(index) }.ok())
.collect();
let boxes: Vec<_> = elements
.iter()
.map(|element| {
let class = unsafe { element.CachedClassName() }
.map(|name| name.to_string())
.unwrap_or_default();
if class.contains(TAB_DRAG_LAYER) {
return Default::default();
}
unsafe { element.CachedBoundingRectangle() }
.map(|rect| (rect.left, rect.top, rect.right, rect.bottom))
.unwrap_or_default()
})
.collect();
smallest_under(&boxes, point.x, point.y).map(|index| elements[index].clone())
}
pub fn element_at_point(x: i32, y: i32) -> Result<Option<UiElement>> {
let uia = automation().map_err(|error| napi::Error::from_reason(error.message()))?;
let point = POINT { x, y };
let hit =
unsafe { uia.ElementFromPoint(point) }.map_err(|error| napi::Error::from_reason(error.message()))?;
let walker = unsafe { uia.ControlViewWalker() }.ok();
let found = walker
.as_ref()
.and_then(|walker| narrowest(&uia, walker, &hit, point))
.unwrap_or(hit);
let mut element = describe(&found);
if element.name.is_none() {
if let Some(walker) = &walker {
element.ancestors = ancestors(walker, &found);
element.children = children(walker, &found);
}
}
Ok(Some(element))
}
pub fn focused_element() -> Result<Option<UiElement>> {
let found = automation()
.and_then(|uia| unsafe { uia.GetFocusedElement() })
.map_err(|error| napi::Error::from_reason(error.message()))?;
Ok(Some(describe(&found)))
}
const NAMED_KEYS: &[(u16, &str)] = &[
(VK_RETURN.0, "Enter"),
(VK_TAB.0, "Tab"),
(VK_ESCAPE.0, "Escape"),
(VK_BACK.0, "Backspace"),
(VK_SPACE.0, "Space"),
(VK_DELETE.0, "Delete"),
(VK_INSERT.0, "Insert"),
(VK_HOME.0, "Home"),
(VK_END.0, "End"),
(VK_PRIOR.0, "PageUp"),
(VK_NEXT.0, "PageDown"),
(VK_UP.0, "ArrowUp"),
(VK_DOWN.0, "ArrowDown"),
(VK_LEFT.0, "ArrowLeft"),
(VK_RIGHT.0, "ArrowRight"),
];
const EXTENDED: u32 = 0xe00;
fn foreground_layout() -> HKL {
let window = unsafe { GetForegroundWindow() };
let thread = unsafe { GetWindowThreadProcessId(window, None) };
unsafe { GetKeyboardLayout(thread) }
}
pub fn key_label(keycode: u32) -> Option<String> {
let extended = keycode >= EXTENDED;
let scancode = if extended {
0xe000 | (keycode - EXTENDED)
} else {
keycode
};
let vk = unsafe { MapVirtualKeyExW(scancode, MAPVK_VSC_TO_VK_EX, Some(foreground_layout())) } as u16;
if vk == 0 {
return None;
}
if let Some((_, name)) = NAMED_KEYS.iter().find(|(key, _)| *key == vk) {
return Some((*name).to_string());
}
if (VK_F1.0..=VK_F24.0).contains(&vk) {
return Some(format!("F{}", vk - VK_F1.0 + 1));
}
if vk < 128 && (vk as u8).is_ascii_alphanumeric() {
return Some((vk as u8 as char).to_string());
}
None
}
fn virtual_key(keycode: u32, layout: HKL) -> u16 {
let extended = keycode >= EXTENDED;
let scancode = if extended {
0xe000 | (keycode - EXTENDED)
} else {
keycode
};
unsafe { MapVirtualKeyExW(scancode, MAPVK_VSC_TO_VK_EX, Some(layout)) as u16 }
}
pub fn resolve_key(keycode: u32, shift: bool, ctrl: bool, alt: bool) -> Option<String> {
let layout = foreground_layout();
let vk = virtual_key(keycode, layout);
if vk == 0 {
return None;
}
let mut state = [0u8; 256];
if shift {
state[VK_SHIFT.0 as usize] = 0x80;
}
if ctrl {
state[VK_CONTROL.0 as usize] = 0x80;
}
if alt {
state[VK_MENU.0 as usize] = 0x80;
}
if unsafe { GetKeyState(VK_CAPITAL.0 as i32) } & 1 == 1 {
state[VK_CAPITAL.0 as usize] = 0x01;
}
let scancode = unsafe { MapVirtualKeyExW(vk as u32, MAPVK_VK_TO_VSC, Some(layout)) };
let mut buffer = [0u16; 8];
let written = unsafe { ToUnicodeEx(vk as u32, scancode, &state, &mut buffer, 0, Some(layout)) };
if written <= 0 {
return None;
}
String::from_utf16(&buffer[..written as usize])
.ok()
.filter(|found| !found.is_empty())
}
pub fn clear_dead_key() {
let layout = foreground_layout();
let vk = VK_SPACE.0 as u32;
let scancode = unsafe { MapVirtualKeyExW(vk, MAPVK_VK_TO_VSC, Some(layout)) };
let state = [0u8; 256];
let mut buffer = [0u16; 8];
for _ in 0..4 {
if unsafe { ToUnicodeEx(vk, scancode, &state, &mut buffer, 0, Some(layout)) } >= 0 {
break;
}
}
}
fn area(rect: &RECT) -> f64 {
f64::from((rect.right - rect.left).max(0)) * f64::from((rect.bottom - rect.top).max(0))
}
fn window_rect(window: HWND) -> Option<RECT> {
let mut rect = RECT::default();
unsafe { GetWindowRect(window, &mut rect) }.ok()?;
Some(rect)
}
fn frame_rect(window: HWND) -> Option<RECT> {
let mut rect = RECT::default();
let framed = unsafe {
DwmGetWindowAttribute(
window,
DWMWA_EXTENDED_FRAME_BOUNDS,
(&mut rect as *mut RECT).cast::<c_void>(),
size_of::<RECT>() as u32,
)
};
if framed.is_ok() && area(&rect) > 0.0 {
Some(rect)
} else {
window_rect(window)
}
}
fn edges(rect: &RECT) -> Edges {
Edges {
left: rect.left,
top: rect.top,
right: rect.right,
bottom: rect.bottom,
}
}
fn fills_its_monitor(window: HWND, bounds: &RECT) -> bool {
let monitor = unsafe { MonitorFromWindow(window, MONITOR_DEFAULTTONEAREST) };
let mut info = MONITORINFO {
cbSize: size_of::<MONITORINFO>() as u32,
..Default::default()
};
unsafe { GetMonitorInfoW(monitor, &mut info) }.as_bool() && fills(edges(bounds), edges(&info.rcWork))
}
fn text_of(read: impl FnOnce(&mut [u16]) -> i32) -> Option<String> {
let mut buffer = [0u16; 512];
let length = read(&mut buffer);
if length <= 0 {
return None;
}
clean(String::from_utf16_lossy(&buffer[..length as usize]))
}
fn style(window: HWND) -> u32 {
unsafe { GetWindowLongPtrW(window, GWL_STYLE) as u32 }
}
fn owned(window: HWND) -> bool {
unsafe { GetWindow(window, GW_OWNER) }.is_ok_and(|owner| !owner.is_invalid())
}
fn cloaked(window: HWND) -> bool {
let mut flag = 0u32;
let read = unsafe {
DwmGetWindowAttribute(
window,
DWMWA_CLOAKED,
(&mut flag as *mut u32).cast::<c_void>(),
size_of::<u32>() as u32,
)
};
read.is_ok() && flag != 0
}
fn popup(window: HWND) -> bool {
let class = text_of(|buffer| unsafe { GetClassNameW(window, buffer) }).unwrap_or_default();
let style = style(window);
let extended = unsafe { GetWindowLongPtrW(window, GWL_EXSTYLE) as u32 };
is_popup(&Traits {
class: &class,
owned: owned(window),
no_activate: extended & WS_EX_NOACTIVATE.0 != 0,
tool_window: extended & WS_EX_TOOLWINDOW.0 != 0,
captionless_popup: style & WS_POPUP.0 != 0 && style & WS_CAPTION.0 != WS_CAPTION.0,
})
}
fn process_of(window: HWND) -> u32 {
let mut process = 0u32;
unsafe { GetWindowThreadProcessId(window, Some(&mut process)) };
process
}
fn main_window_of(process: u32) -> Option<HWND> {
struct Search {
process: u32,
best: Option<(f64, HWND)>,
}
unsafe extern "system" fn visit(window: HWND, data: LPARAM) -> BOOL {
let search = unsafe { &mut *(data.0 as *mut Search) };
let style = style(window);
let candidate = process_of(window) == search.process
&& unsafe { IsWindowVisible(window) }.as_bool()
&& !cloaked(window)
&& !owned(window)
&& style & WS_CAPTION.0 != 0
&& style & WS_SYSMENU.0 != 0;
if let Some(rect) = window_rect(window).filter(|_| candidate) {
let size = area(&rect);
if search.best.is_none_or(|(largest, _)| size > largest) {
search.best = Some((size, window));
}
}
true.into()
}
let mut search = Search { process, best: None };
let _ = unsafe { EnumWindows(Some(visit), LPARAM(&mut search as *mut Search as isize)) };
search.best.map(|(_, window)| window)
}
fn process_path(process: u32) -> Option<String> {
let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, process) }.ok()?;
let mut buffer = [0u16; 1024];
let mut length = buffer.len() as u32;
let named = unsafe {
QueryFullProcessImageNameW(
handle,
PROCESS_NAME_WIN32,
PWSTR(buffer.as_mut_ptr()),
&mut length,
)
};
let _ = unsafe { CloseHandle(handle) };
named.ok()?;
clean(String::from_utf16_lossy(&buffer[..length as usize]))
}
fn wide(text: &str) -> Vec<u16> {
text.encode_utf16().chain(Some(0)).collect()
}
fn file_description(path: &str) -> Option<String> {
let file = wide(path);
let size = unsafe { GetFileVersionInfoSizeW(PCWSTR(file.as_ptr()), None) };
if size == 0 {
return None;
}
let mut data = vec![0u8; size as usize];
unsafe { GetFileVersionInfoW(PCWSTR(file.as_ptr()), None, size, data.as_mut_ptr().cast()) }.ok()?;
let query = |key: &str| -> Option<(*const u16, usize)> {
let key = wide(key);
let mut found: *mut c_void = std::ptr::null_mut();
let mut length = 0u32;
let hit = unsafe {
VerQueryValueW(
data.as_ptr().cast(),
PCWSTR(key.as_ptr()),
&mut found,
&mut length,
)
};
(hit.as_bool() && length > 0 && !found.is_null()).then_some((found as *const u16, length as usize))
};
let (translation, _) = query("\\VarFileInfo\\Translation")?;
let (language, codepage) = unsafe { (*translation, *translation.add(1)) };
let (text, length) = query(&format!(
"\\StringFileInfo\\{language:04x}{codepage:04x}\\FileDescription"
))?;
let characters = unsafe { std::slice::from_raw_parts(text, length) };
clean(
String::from_utf16_lossy(characters)
.trim_end_matches('\0')
.to_string(),
)
}
pub fn active_window() -> Option<ActiveWindow> {
framed(unsafe { GetForegroundWindow() })
}
pub fn window_at(x: i32, y: i32) -> Option<ActiveWindow> {
let hit = unsafe { WindowFromPoint(POINT { x, y }) };
if hit.is_invalid() {
return None;
}
framed(unsafe { GetAncestor(hit, GA_ROOT) })
}
fn framed(front: HWND) -> Option<ActiveWindow> {
if front.is_invalid() {
return None;
}
let process = process_of(front);
let bounds = frame_rect(front)?;
let window = if !fills_its_monitor(front, &bounds) && popup(front) {
main_window_of(process).unwrap_or(front)
} else {
front
};
let rect = if window == front {
bounds
} else {
frame_rect(window)?
};
let path = process_path(process);
let app_name = path
.as_deref()
.and_then(file_description)
.or_else(|| {
path
.as_deref()
.and_then(|found| Path::new(found).file_stem())
.map(|stem| stem.to_string_lossy().into_owned())
})
.unwrap_or_default();
Some(ActiveWindow {
title: text_of(|buffer| unsafe { GetWindowTextW(window, buffer) }),
app_name,
app_path: path,
x: f64::from(rect.left),
y: f64::from(rect.top),
width: f64::from(rect.right - rect.left),
height: f64::from(rect.bottom - rect.top),
on_menu: false,
})
}