Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

fix(desktop): capture typing in browser web forms and password fields on macOS

Luis Ortiz30aeb8887d6f
diff --git a/apps/desktop/src/main/capture/element.ts b/apps/desktop/src/main/capture/element.ts
index 6e6e78c..b21f114 100644
--- a/apps/desktop/src/main/capture/element.ts
+++ b/apps/desktop/src/main/capture/element.ts
@@ -12,6 +12,7 @@ export interface ScreenElement {
ariaLabel: string | null;
altText: string | null;
password: boolean;
+ length?: number | null;
rect: { x: number; y: number; width: number; height: number } | null;
ancestors: { role: string | null; name: string | null }[];
children: { role: string | null; name: string | null }[];
@@ -50,6 +51,7 @@ function describe(found: UiElement): ScreenElement {
ariaLabel: found.name ?? null,
altText: found.helpText ?? null,
password: found.isPassword,
+ length: found.valueLength ?? null,
rect: toDip(found.rect ?? null),
ancestors: (found.ancestors ?? []).map(node),
children: (found.children ?? []).map(node),
@@ -91,6 +93,11 @@ export async function clearDeadKey(): Promise<void> {
native?.clearDeadKey();
}
+export async function releaseWebContent(): Promise<void> {
+ const native = await load();
+ native?.releaseWebContent();
+}
+
export async function elementLookupAvailable(): Promise<boolean> {
const native = await load();
return native?.isSupported() ?? false;
diff --git a/apps/desktop/src/main/capture/native.ts b/apps/desktop/src/main/capture/native.ts
index 2ed00e6..aead3f9 100644
--- a/apps/desktop/src/main/capture/native.ts
+++ b/apps/desktop/src/main/capture/native.ts
@@ -8,6 +8,7 @@ export type Addon = {
keyLabel(keycode: number): string | null;
resolveKey(keycode: number, shift: boolean, ctrl: boolean, alt: boolean): string | null;
clearDeadKey(): void;
+ releaseWebContent(): void;
isSupported(): boolean;
startInputHook(callback: (event: HookEvent) => void): void;
};
diff --git a/apps/desktop/src/main/capture/recorder.ts b/apps/desktop/src/main/capture/recorder.ts
index f0a0f1c..0ce5bac 100644
--- a/apps/desktop/src/main/capture/recorder.ts
+++ b/apps/desktop/src/main/capture/recorder.ts
@@ -9,6 +9,7 @@ import {
focusedField,
isTextField,
keyLabel,
+ releaseWebContent,
resolveKey,
type ScreenElement,
} from './element';
@@ -81,6 +82,7 @@ export interface RecorderHooks {
label?: (keycode: number) => Promise<string | null>;
resolve?: (keycode: number, shift: boolean, ctrl: boolean, alt: boolean) => Promise<string | null>;
reset?: () => Promise<void>;
+ release?: () => Promise<void>;
drained?: () => void;
progress?: (fraction: number, ms: number) => void;
aimed?: (aim: { x: number; y: number; aspect: number }) => void;
@@ -90,6 +92,22 @@ export interface RecorderHooks {
export type RecorderStart = { ok: true } | { ok: false; reason: string; detail: string };
const CAPTURE_ESTIMATE_MS = 450;
+const SECURE_INPUT_HIDES_KEYS = process.platform === 'darwin';
+const FOCUS_SETTLE_MS = 200;
+const PASSWORD_POLL_MS = 400;
+
+interface PasswordWatch {
+ field: ScreenElement;
+ start: number;
+ last: number;
+ shot: Promise<Frame> | null;
+ timer: NodeJS.Timeout;
+ idle: NodeJS.Timeout | null;
+}
+
+function sameRect(a: ScreenElement['rect'], b: ScreenElement['rect']): boolean {
+ return a?.x === b?.x && a?.y === b?.y && a?.width === b?.width && a?.height === b?.height;
+}
function delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
@@ -212,6 +230,7 @@ export class DesktopRecorder {
private snapshotTimer: NodeJS.Timeout | null = null;
private appending: Promise<unknown> = Promise.resolve();
private idle: NodeJS.Timeout | null = null;
+ private passwordWatch: PasswordWatch | null = null;
private readonly grab: (point: Point) => Promise<Frame>;
private readonly settings: () => CaptureSettings;
private readonly ignores: (point: Point) => boolean;
@@ -221,6 +240,7 @@ export class DesktopRecorder {
private readonly label: (keycode: number) => Promise<string | null>;
private readonly resolve: (keycode: number, shift: boolean, ctrl: boolean, alt: boolean) => Promise<string | null>;
private readonly reset: () => Promise<void>;
+ private readonly release: () => Promise<void>;
private readonly drained: () => void;
private readonly progress: (fraction: number, ms: number) => void;
private readonly aimed: (aim: { x: number; y: number; aspect: number }) => void;
@@ -242,6 +262,7 @@ export class DesktopRecorder {
this.label = hooks.label ?? keyLabel;
this.resolve = hooks.resolve ?? resolveKey;
this.reset = hooks.reset ?? clearDeadKey;
+ this.release = hooks.release ?? releaseWebContent;
this.drained = hooks.drained ?? (() => {});
this.progress = hooks.progress ?? (() => {});
this.aimed = hooks.aimed ?? (() => {});
@@ -272,6 +293,7 @@ export class DesktopRecorder {
stop(): void {
this.commitTyping();
this.hook.stop();
+ void this.release().catch(() => undefined);
this.running = false;
this.paused = false;
}
@@ -298,12 +320,79 @@ export class DesktopRecorder {
private clickAt(action: string, point: Point): void {
this.commitTyping();
+ this.watchPassword();
const element = this.lookup(point);
const frame = this.shoot(point);
const place = this.windowAt(point).catch(() => focusedWindow());
this.enqueue(() => this.write(action, point, element, frame, undefined, place));
}
+ private watchPassword(): void {
+ if (!SECURE_INPUT_HIDES_KEYS || !this.settings().recordTyping) return;
+ const settle = setTimeout(async () => {
+ const field = await this.focused().catch(() => null);
+ if (!field?.password || this.passwordWatch || !this.isRecording) return;
+ const length = field.length ?? 0;
+ const watch: PasswordWatch = {
+ field,
+ start: length,
+ last: length,
+ shot: null,
+ timer: setInterval(() => void this.pollPassword(watch), PASSWORD_POLL_MS),
+ idle: null,
+ };
+ watch.timer.unref?.();
+ this.passwordWatch = watch;
+ }, FOCUS_SETTLE_MS);
+ settle.unref?.();
+ }
+
+ private async pollPassword(watch: PasswordWatch): Promise<void> {
+ const now = await this.focused().catch(() => null);
+ if (this.passwordWatch !== watch || !now) return;
+ if (!now.password || !sameRect(now.rect, watch.field.rect)) {
+ this.closePassword();
+ return;
+ }
+ const length = now.length ?? watch.last;
+ if (length === watch.last) return;
+ watch.last = length;
+ watch.shot = this.grab(centreOf(watch.field) ?? cursorPoint());
+ watch.shot.catch(() => undefined);
+ if (watch.idle) clearTimeout(watch.idle);
+ watch.idle = setTimeout(() => {
+ if (this.passwordWatch === watch) this.writePassword(watch, Promise.resolve(watch.last));
+ }, this.settings().typingDebounceMs);
+ watch.idle.unref?.();
+ }
+
+ private writePassword(watch: PasswordWatch, final: Promise<number>): void {
+ if (watch.idle) clearTimeout(watch.idle);
+ watch.idle = null;
+ const { start, field } = watch;
+ const centre = centreOf(field) ?? cursorPoint();
+ const shot = watch.shot ?? this.grab(centre);
+ shot.catch(() => undefined);
+ watch.shot = null;
+ watch.start = watch.last;
+ this.enqueue(async () => {
+ if ((await final) === start) return;
+ const place = this.windowAt(centre).catch(() => focusedWindow());
+ await this.write('input', centre, Promise.resolve(field), shot, undefined, place);
+ });
+ }
+
+ private closePassword(): void {
+ const watch = this.passwordWatch;
+ if (!watch) return;
+ this.passwordWatch = null;
+ clearInterval(watch.timer);
+ const final = this.focused()
+ .catch(() => null)
+ .then((now) => (now?.password && sameRect(now.rect, watch.field.rect) ? (now.length ?? watch.last) : watch.last));
+ this.writePassword(watch, final);
+ }
+
private onKey(action: KeyAction): void {
if (MODIFIER_KEYS.has(action.keycode)) return;
const settings = this.settings();
@@ -325,6 +414,7 @@ export class DesktopRecorder {
const submitted = this.typing;
this.commitTyping(true);
+ if (action.keycode === KEY.tab) this.watchPassword();
const at = Date.now();
const repeat = isRepeatKey(this.lastKey, action.keycode, at);
@@ -357,6 +447,7 @@ export class DesktopRecorder {
}
private commitTyping(live = false): void {
+ this.closePassword();
if (this.idle) clearTimeout(this.idle);
if (this.snapshotTimer) clearTimeout(this.snapshotTimer);
this.idle = null;
@@ -444,6 +535,7 @@ export class DesktopRecorder {
const shot = frame ?? this.shoot(where);
const place = centre ? this.windowAt(centre).catch(() => focusedWindow()) : undefined;
if (field?.password) {
+ if (SECURE_INPUT_HIDES_KEYS) return;
await this.write('input', where, Promise.resolve(field), shot, undefined, place);
return;
}
diff --git a/packages/capture-native/index.d.ts b/packages/capture-native/index.d.ts
index c95ac6c..8209a93 100644
--- a/packages/capture-native/index.d.ts
+++ b/packages/capture-native/index.d.ts
@@ -24,6 +24,8 @@ export interface ActiveWindow {
export declare function clearDeadKey(): void
+export declare function releaseWebContent(): void
+
export declare function elementAtPoint(x: number, y: number): Promise<UiElement | null>
export interface ElementNode {
@@ -68,6 +70,7 @@ export interface UiElement {
value?: string
helpText?: string
isPassword: boolean
+ valueLength?: number
rect?: ElementRect
ancestors: Array<ElementNode>
children: Array<ElementNode>
diff --git a/packages/capture-native/src/lib.rs b/packages/capture-native/src/lib.rs
index 45e3e3b..8b5bebc 100644
--- a/packages/capture-native/src/lib.rs
+++ b/packages/capture-native/src/lib.rs
@@ -43,6 +43,7 @@ pub struct UiElement {
pub value: Option<String>,
pub help_text: Option<String>,
pub is_password: bool,
+ pub value_length: Option<u32>,
pub rect: Option<ElementRect>,
pub ancestors: Vec<ElementNode>,
pub children: Vec<ElementNode>,
@@ -171,6 +172,12 @@ pub fn clear_dead_key() {
platform::clear_dead_key();
}
+#[napi]
+pub fn release_web_content() {
+ #[cfg(target_os = "macos")]
+ platform::release_web_content();
+}
+
#[napi]
pub fn is_supported() -> bool {
cfg!(any(windows, target_os = "macos"))
diff --git a/packages/capture-native/src/mac.rs b/packages/capture-native/src/mac.rs
index 5319a0d..49de5cf 100644
--- a/packages/capture-native/src/mac.rs
+++ b/packages/capture-native/src/mac.rs
@@ -57,6 +57,7 @@ extern "C" {
fn CFRelease(value: CFTypeRef);
fn CFGetTypeID(value: CFTypeRef) -> usize;
fn CFStringGetTypeID() -> usize;
+ fn CFNumberGetTypeID() -> usize;
fn CFArrayGetTypeID() -> usize;
fn CFStringCreateWithBytes(
allocator: *const c_void,
@@ -74,6 +75,7 @@ extern "C" {
fn CFNumberGetValue(number: CFTypeRef, kind: isize, value: *mut c_void) -> Boolean;
fn CFDataGetBytePtr(data: CFTypeRef) -> *const u8;
static kCFBooleanTrue: CFTypeRef;
+ static kCFBooleanFalse: CFTypeRef;
}
#[link(name = "ApplicationServices", kind = "framework")]
@@ -219,6 +221,17 @@ fn secure(element: AXUIElementRef) -> bool {
text(element, "AXSubrole").as_deref() == Some("AXSecureTextField")
}
+fn secure_length(element: AXUIElementRef) -> Option<u32> {
+ let counted = attribute(element, "AXNumberOfCharacters")
+ .filter(|found| unsafe { CFGetTypeID(found.0) == CFNumberGetTypeID() })
+ .and_then(|found| {
+ let mut out = 0i32;
+ (unsafe { CFNumberGetValue(found.0, NUMBER_INT32, (&mut out as *mut i32).cast()) } != 0)
+ .then_some(out.max(0) as u32)
+ });
+ counted.or_else(|| text(element, "AXValue").map(|value| value.chars().count() as u32))
+}
+
fn label_of(element: AXUIElementRef) -> Option<String> {
text(element, "AXTitle")
.or_else(|| text(element, "AXDescription"))
@@ -241,6 +254,7 @@ fn describe(element: AXUIElementRef) -> UiElement {
},
help_text: text(element, "AXHelp").or_else(|| text(element, "AXPlaceholderValue")),
is_password,
+ value_length: if is_password { secure_length(element) } else { None },
rect: rect_of(element),
ancestors: Vec::new(),
children: Vec::new(),
@@ -300,6 +314,8 @@ fn promoted(hit: Owned) -> Owned {
}
static OPENED: Mutex<Option<HashSet<i32>>> = Mutex::new(None);
+static ENHANCED: Mutex<Vec<i32>> = Mutex::new(Vec::new());
+const ENHANCED_UI: &str = "AXEnhancedUserInterface";
fn pid_of(element: AXUIElementRef) -> Option<i32> {
let mut pid = 0;
@@ -321,8 +337,32 @@ fn open_web_content(element: AXUIElementRef) -> bool {
let Some(app) = Owned::new(unsafe { AXUIElementCreateApplication(pid) }) else {
return false;
};
- let key = cf_string("AXManualAccessibility");
- unsafe { AXUIElementSetAttributeValue(app.0, key.0, kCFBooleanTrue) == 0 }
+ let manual = cf_string("AXManualAccessibility");
+ let opened = unsafe { AXUIElementSetAttributeValue(app.0, manual.0, kCFBooleanTrue) == 0 };
+ let already = attribute(app.0, ENHANCED_UI).is_some_and(|value| value.0 == unsafe { kCFBooleanTrue });
+ let enhanced = cf_string(ENHANCED_UI);
+ if !already && unsafe { AXUIElementSetAttributeValue(app.0, enhanced.0, kCFBooleanTrue) == 0 } {
+ ENHANCED
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ .push(pid);
+ return true;
+ }
+ opened
+}
+
+pub fn release_web_content() {
+ OPENED
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ .take();
+ let pids = std::mem::take(&mut *ENHANCED.lock().unwrap_or_else(|poisoned| poisoned.into_inner()));
+ let key = cf_string(ENHANCED_UI);
+ for pid in pids {
+ if let Some(app) = Owned::new(unsafe { AXUIElementCreateApplication(pid) }) {
+ unsafe { AXUIElementSetAttributeValue(app.0, key.0, kCFBooleanFalse) };
+ }
+ }
}
fn system() -> Option<Owned> {
diff --git a/packages/capture-native/src/win.rs b/packages/capture-native/src/win.rs
index 30299ad..36b2243 100644
--- a/packages/capture-native/src/win.rs
+++ b/packages/capture-native/src/win.rs
@@ -141,6 +141,7 @@ fn describe(found: &IUIAutomationElement) -> UiElement {
is_password: unsafe { found.CurrentIsPassword() }
.map(|flag| flag.as_bool())
.unwrap_or(false),
+ value_length: None,
rect: rect_of(found),
ancestors: Vec::new(),
children: Vec::new(),