π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
fix(desktop): capture typing in browser web forms and password fields on macOS
Luis Ortiz30aeb8887d6f
diff --git a/apps/desktop/src/main/capture/element.ts b/apps/desktop/src/main/capture/element.tsindex 6e6e78c..b21f114 100644--- a/apps/desktop/src/main/capture/element.ts+++ b/apps/desktop/src/main/capture/element.ts@@ -12,6 +12,7 @@ export interface ScreenElement {ariaLabel: string | null;altText: string | null;password: boolean;+ length?: number | null;rect: { x: number; y: number; width: number; height: number } | null;ancestors: { role: string | null; name: string | null }[];children: { role: string | null; name: string | null }[];@@ -50,6 +51,7 @@ function describe(found: UiElement): ScreenElement {ariaLabel: found.name ?? null,altText: found.helpText ?? null,password: found.isPassword,+ length: found.valueLength ?? null,rect: toDip(found.rect ?? null),ancestors: (found.ancestors ?? []).map(node),children: (found.children ?? []).map(node),@@ -91,6 +93,11 @@ export async function clearDeadKey(): Promise<void> {native?.clearDeadKey();}+export async function releaseWebContent(): Promise<void> {+ const native = await load();+ native?.releaseWebContent();+}+export async function elementLookupAvailable(): Promise<boolean> {const native = await load();return native?.isSupported() ?? false;diff --git a/apps/desktop/src/main/capture/native.ts b/apps/desktop/src/main/capture/native.tsindex 2ed00e6..aead3f9 100644--- a/apps/desktop/src/main/capture/native.ts+++ b/apps/desktop/src/main/capture/native.ts@@ -8,6 +8,7 @@ export type Addon = {keyLabel(keycode: number): string | null;resolveKey(keycode: number, shift: boolean, ctrl: boolean, alt: boolean): string | null;clearDeadKey(): void;+ releaseWebContent(): void;isSupported(): boolean;startInputHook(callback: (event: HookEvent) => void): void;};diff --git a/apps/desktop/src/main/capture/recorder.ts b/apps/desktop/src/main/capture/recorder.tsindex f0a0f1c..0ce5bac 100644--- a/apps/desktop/src/main/capture/recorder.ts+++ b/apps/desktop/src/main/capture/recorder.ts@@ -9,6 +9,7 @@ import {focusedField,isTextField,keyLabel,+ releaseWebContent,resolveKey,type ScreenElement,} from './element';@@ -81,6 +82,7 @@ export interface RecorderHooks {label?: (keycode: number) => Promise<string | null>;resolve?: (keycode: number, shift: boolean, ctrl: boolean, alt: boolean) => Promise<string | null>;reset?: () => Promise<void>;+ release?: () => Promise<void>;drained?: () => void;progress?: (fraction: number, ms: number) => void;aimed?: (aim: { x: number; y: number; aspect: number }) => void;@@ -90,6 +92,22 @@ export interface RecorderHooks {export type RecorderStart = { ok: true } | { ok: false; reason: string; detail: string };const CAPTURE_ESTIMATE_MS = 450;+const SECURE_INPUT_HIDES_KEYS = process.platform === 'darwin';+const FOCUS_SETTLE_MS = 200;+const PASSWORD_POLL_MS = 400;++interface PasswordWatch {+ field: ScreenElement;+ start: number;+ last: number;+ shot: Promise<Frame> | null;+ timer: NodeJS.Timeout;+ idle: NodeJS.Timeout | null;+}++function sameRect(a: ScreenElement['rect'], b: ScreenElement['rect']): boolean {+ return a?.x === b?.x && a?.y === b?.y && a?.width === b?.width && a?.height === b?.height;+}function delay(ms: number): Promise<void> {return new Promise((resolve) => setTimeout(resolve, ms));@@ -212,6 +230,7 @@ export class DesktopRecorder {private snapshotTimer: NodeJS.Timeout | null = null;private appending: Promise<unknown> = Promise.resolve();private idle: NodeJS.Timeout | null = null;+ private passwordWatch: PasswordWatch | null = null;private readonly grab: (point: Point) => Promise<Frame>;private readonly settings: () => CaptureSettings;private readonly ignores: (point: Point) => boolean;@@ -221,6 +240,7 @@ export class DesktopRecorder {private readonly label: (keycode: number) => Promise<string | null>;private readonly resolve: (keycode: number, shift: boolean, ctrl: boolean, alt: boolean) => Promise<string | null>;private readonly reset: () => Promise<void>;+ private readonly release: () => Promise<void>;private readonly drained: () => void;private readonly progress: (fraction: number, ms: number) => void;private readonly aimed: (aim: { x: number; y: number; aspect: number }) => void;@@ -242,6 +262,7 @@ export class DesktopRecorder {this.label = hooks.label ?? keyLabel;this.resolve = hooks.resolve ?? resolveKey;this.reset = hooks.reset ?? clearDeadKey;+ this.release = hooks.release ?? releaseWebContent;this.drained = hooks.drained ?? (() => {});this.progress = hooks.progress ?? (() => {});this.aimed = hooks.aimed ?? (() => {});@@ -272,6 +293,7 @@ export class DesktopRecorder {stop(): void {this.commitTyping();this.hook.stop();+ void this.release().catch(() => undefined);this.running = false;this.paused = false;}@@ -298,12 +320,79 @@ export class DesktopRecorder {private clickAt(action: string, point: Point): void {this.commitTyping();+ this.watchPassword();const element = this.lookup(point);const frame = this.shoot(point);const place = this.windowAt(point).catch(() => focusedWindow());this.enqueue(() => this.write(action, point, element, frame, undefined, place));}+ private watchPassword(): void {+ if (!SECURE_INPUT_HIDES_KEYS || !this.settings().recordTyping) return;+ const settle = setTimeout(async () => {+ const field = await this.focused().catch(() => null);+ if (!field?.password || this.passwordWatch || !this.isRecording) return;+ const length = field.length ?? 0;+ const watch: PasswordWatch = {+ field,+ start: length,+ last: length,+ shot: null,+ timer: setInterval(() => void this.pollPassword(watch), PASSWORD_POLL_MS),+ idle: null,+ };+ watch.timer.unref?.();+ this.passwordWatch = watch;+ }, FOCUS_SETTLE_MS);+ settle.unref?.();+ }++ private async pollPassword(watch: PasswordWatch): Promise<void> {+ const now = await this.focused().catch(() => null);+ if (this.passwordWatch !== watch || !now) return;+ if (!now.password || !sameRect(now.rect, watch.field.rect)) {+ this.closePassword();+ return;+ }+ const length = now.length ?? watch.last;+ if (length === watch.last) return;+ watch.last = length;+ watch.shot = this.grab(centreOf(watch.field) ?? cursorPoint());+ watch.shot.catch(() => undefined);+ if (watch.idle) clearTimeout(watch.idle);+ watch.idle = setTimeout(() => {+ if (this.passwordWatch === watch) this.writePassword(watch, Promise.resolve(watch.last));+ }, this.settings().typingDebounceMs);+ watch.idle.unref?.();+ }++ private writePassword(watch: PasswordWatch, final: Promise<number>): void {+ if (watch.idle) clearTimeout(watch.idle);+ watch.idle = null;+ const { start, field } = watch;+ const centre = centreOf(field) ?? cursorPoint();+ const shot = watch.shot ?? this.grab(centre);+ shot.catch(() => undefined);+ watch.shot = null;+ watch.start = watch.last;+ this.enqueue(async () => {+ if ((await final) === start) return;+ const place = this.windowAt(centre).catch(() => focusedWindow());+ await this.write('input', centre, Promise.resolve(field), shot, undefined, place);+ });+ }++ private closePassword(): void {+ const watch = this.passwordWatch;+ if (!watch) return;+ this.passwordWatch = null;+ clearInterval(watch.timer);+ const final = this.focused()+ .catch(() => null)+ .then((now) => (now?.password && sameRect(now.rect, watch.field.rect) ? (now.length ?? watch.last) : watch.last));+ this.writePassword(watch, final);+ }+private onKey(action: KeyAction): void {if (MODIFIER_KEYS.has(action.keycode)) return;const settings = this.settings();@@ -325,6 +414,7 @@ export class DesktopRecorder {const submitted = this.typing;this.commitTyping(true);+ if (action.keycode === KEY.tab) this.watchPassword();const at = Date.now();const repeat = isRepeatKey(this.lastKey, action.keycode, at);@@ -357,6 +447,7 @@ export class DesktopRecorder {}private commitTyping(live = false): void {+ this.closePassword();if (this.idle) clearTimeout(this.idle);if (this.snapshotTimer) clearTimeout(this.snapshotTimer);this.idle = null;@@ -444,6 +535,7 @@ export class DesktopRecorder {const shot = frame ?? this.shoot(where);const place = centre ? this.windowAt(centre).catch(() => focusedWindow()) : undefined;if (field?.password) {+ if (SECURE_INPUT_HIDES_KEYS) return;await this.write('input', where, Promise.resolve(field), shot, undefined, place);return;}diff --git a/packages/capture-native/index.d.ts b/packages/capture-native/index.d.tsindex c95ac6c..8209a93 100644--- a/packages/capture-native/index.d.ts+++ b/packages/capture-native/index.d.ts@@ -24,6 +24,8 @@ export interface ActiveWindow {export declare function clearDeadKey(): void+export declare function releaseWebContent(): void+export declare function elementAtPoint(x: number, y: number): Promise<UiElement | null>export interface ElementNode {@@ -68,6 +70,7 @@ export interface UiElement {value?: stringhelpText?: stringisPassword: boolean+ valueLength?: numberrect?: ElementRectancestors: Array<ElementNode>children: Array<ElementNode>diff --git a/packages/capture-native/src/lib.rs b/packages/capture-native/src/lib.rsindex 45e3e3b..8b5bebc 100644--- a/packages/capture-native/src/lib.rs+++ b/packages/capture-native/src/lib.rs@@ -43,6 +43,7 @@ pub struct UiElement {pub value: Option<String>,pub help_text: Option<String>,pub is_password: bool,+ pub value_length: Option<u32>,pub rect: Option<ElementRect>,pub ancestors: Vec<ElementNode>,pub children: Vec<ElementNode>,@@ -171,6 +172,12 @@ pub fn clear_dead_key() {platform::clear_dead_key();}+#[napi]+pub fn release_web_content() {+ #[cfg(target_os = "macos")]+ platform::release_web_content();+}+#[napi]pub fn is_supported() -> bool {cfg!(any(windows, target_os = "macos"))diff --git a/packages/capture-native/src/mac.rs b/packages/capture-native/src/mac.rsindex 5319a0d..49de5cf 100644--- a/packages/capture-native/src/mac.rs+++ b/packages/capture-native/src/mac.rs@@ -57,6 +57,7 @@ extern "C" {fn CFRelease(value: CFTypeRef);fn CFGetTypeID(value: CFTypeRef) -> usize;fn CFStringGetTypeID() -> usize;+ fn CFNumberGetTypeID() -> usize;fn CFArrayGetTypeID() -> usize;fn CFStringCreateWithBytes(allocator: *const c_void,@@ -74,6 +75,7 @@ extern "C" {fn CFNumberGetValue(number: CFTypeRef, kind: isize, value: *mut c_void) -> Boolean;fn CFDataGetBytePtr(data: CFTypeRef) -> *const u8;static kCFBooleanTrue: CFTypeRef;+ static kCFBooleanFalse: CFTypeRef;}#[link(name = "ApplicationServices", kind = "framework")]@@ -219,6 +221,17 @@ fn secure(element: AXUIElementRef) -> bool {text(element, "AXSubrole").as_deref() == Some("AXSecureTextField")}+fn secure_length(element: AXUIElementRef) -> Option<u32> {+ let counted = attribute(element, "AXNumberOfCharacters")+ .filter(|found| unsafe { CFGetTypeID(found.0) == CFNumberGetTypeID() })+ .and_then(|found| {+ let mut out = 0i32;+ (unsafe { CFNumberGetValue(found.0, NUMBER_INT32, (&mut out as *mut i32).cast()) } != 0)+ .then_some(out.max(0) as u32)+ });+ counted.or_else(|| text(element, "AXValue").map(|value| value.chars().count() as u32))+}+fn label_of(element: AXUIElementRef) -> Option<String> {text(element, "AXTitle").or_else(|| text(element, "AXDescription"))@@ -241,6 +254,7 @@ fn describe(element: AXUIElementRef) -> UiElement {},help_text: text(element, "AXHelp").or_else(|| text(element, "AXPlaceholderValue")),is_password,+ value_length: if is_password { secure_length(element) } else { None },rect: rect_of(element),ancestors: Vec::new(),children: Vec::new(),@@ -300,6 +314,8 @@ fn promoted(hit: Owned) -> Owned {}static OPENED: Mutex<Option<HashSet<i32>>> = Mutex::new(None);+static ENHANCED: Mutex<Vec<i32>> = Mutex::new(Vec::new());+const ENHANCED_UI: &str = "AXEnhancedUserInterface";fn pid_of(element: AXUIElementRef) -> Option<i32> {let mut pid = 0;@@ -321,8 +337,32 @@ fn open_web_content(element: AXUIElementRef) -> bool {let Some(app) = Owned::new(unsafe { AXUIElementCreateApplication(pid) }) else {return false;};- let key = cf_string("AXManualAccessibility");- unsafe { AXUIElementSetAttributeValue(app.0, key.0, kCFBooleanTrue) == 0 }+ let manual = cf_string("AXManualAccessibility");+ let opened = unsafe { AXUIElementSetAttributeValue(app.0, manual.0, kCFBooleanTrue) == 0 };+ let already = attribute(app.0, ENHANCED_UI).is_some_and(|value| value.0 == unsafe { kCFBooleanTrue });+ let enhanced = cf_string(ENHANCED_UI);+ if !already && unsafe { AXUIElementSetAttributeValue(app.0, enhanced.0, kCFBooleanTrue) == 0 } {+ ENHANCED+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner())+ .push(pid);+ return true;+ }+ opened+}++pub fn release_web_content() {+ OPENED+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner())+ .take();+ let pids = std::mem::take(&mut *ENHANCED.lock().unwrap_or_else(|poisoned| poisoned.into_inner()));+ let key = cf_string(ENHANCED_UI);+ for pid in pids {+ if let Some(app) = Owned::new(unsafe { AXUIElementCreateApplication(pid) }) {+ unsafe { AXUIElementSetAttributeValue(app.0, key.0, kCFBooleanFalse) };+ }+ }}fn system() -> Option<Owned> {diff --git a/packages/capture-native/src/win.rs b/packages/capture-native/src/win.rsindex 30299ad..36b2243 100644--- a/packages/capture-native/src/win.rs+++ b/packages/capture-native/src/win.rs@@ -141,6 +141,7 @@ fn describe(found: &IUIAutomationElement) -> UiElement {is_password: unsafe { found.CurrentIsPassword() }.map(|flag| flag.as_bool()).unwrap_or(false),+ value_length: None,rect: rect_of(found),ancestors: Vec::new(),children: Vec::new(),