Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

fix(desktop): frame each click by the window or area it belongs to

Luis Ortiz68392d4c4cb6
diff --git a/apps/desktop/scripts/check-pipeline.ts b/apps/desktop/scripts/check-pipeline.ts
index 5d02555..0b2fdfc 100644
--- a/apps/desktop/scripts/check-pipeline.ts
+++ b/apps/desktop/scripts/check-pipeline.ts
@@ -186,8 +186,33 @@ app.whenReady().then(async () => {
sameRect(frameFor('window', insidePoint, region, elsewhere), display.bounds) &&
sameRect(frameFor('window', insidePoint, region, null), display.bounds) &&
sameRect(frameFor('region', insidePoint, region, windowRect), region) &&
- sameRect(frameFor('region', outsidePoint, region, windowRect), display.bounds),
- detail: 'screen takes the display, window takes the window and falls back twice, region takes the region',
+ sameRect(frameFor('region', outsidePoint, region, windowRect), region),
+ detail: 'screen takes the display, window takes the window and falls back twice, region always takes the region',
+ });
+
+ let framedBy: CaptureRequest | null = null;
+ const clickedWindow = { x: region.x + 40, y: region.y + 30, width: 300, height: 200 };
+ const windowed = new DesktopRecorder(
+ () => region,
+ (fn) => fn(),
+ (request) => {
+ framedBy = request;
+ return Promise.resolve(null);
+ },
+ {
+ grab: async () => syntheticDisplay,
+ settings: () => ({ ...DEFAULT_CAPTURE_SETTINGS, captureMode: 'window', showCursor: false }),
+ lookup: () => Promise.resolve(null),
+ windowAt: () =>
+ Promise.resolve({ ok: true, window: { title: 'Left pane', app: { name: 'Explorer' }, bounds: clickedWindow } }),
+ },
+ );
+ await windowed.capture({ x: clickedWindow.x + 20, y: clickedWindow.y + 20 });
+ const windowShot = (framedBy as CaptureRequest | null)?.image;
+ results.push({
+ name: 'window mode crops to the window that was clicked',
+ ok: windowShot?.width === Math.round(clickedWindow.width * display.scaleFactor),
+ detail: `${windowShot?.width ?? 0} px wide for a ${clickedWindow.width} px window`,
});
const control: ScreenElement = {
diff --git a/apps/desktop/src/main/capture/element.ts b/apps/desktop/src/main/capture/element.ts
index 494e34f..6e6e78c 100644
--- a/apps/desktop/src/main/capture/element.ts
+++ b/apps/desktop/src/main/capture/element.ts
@@ -1,6 +1,7 @@
import type { ElementNode, UiElement } from '@mimik/capture-native';
import { screen } from 'electron';
import { toDip } from './focused-window';
+import { loadNative as load } from './native';
const LOOKUP_TIMEOUT_MS = 1500;
@@ -18,28 +19,6 @@ export interface ScreenElement {
const TEXT_ROLES = new Set(['textbox', 'combobox', 'document']);
-type Addon = {
- elementAtPoint(x: number, y: number): Promise<UiElement | null>;
- focusedElement(): Promise<UiElement | null>;
- keyLabel(keycode: number): string | null;
- resolveKey(keycode: number, shift: boolean, ctrl: boolean, alt: boolean): string | null;
- clearDeadKey(): void;
- isSupported(): boolean;
-};
-
-let addon: Addon | null | undefined;
-
-async function load(): Promise<Addon | null> {
- if (addon === undefined) {
- try {
- addon = (await import('@mimik/capture-native')) as unknown as Addon;
- } catch {
- addon = null;
- }
- }
- return addon;
-}
-
function within<T>(work: Promise<T>, ms: number): Promise<T | null> {
return new Promise((resolve) => {
const timer = setTimeout(() => resolve(null), ms);
diff --git a/apps/desktop/src/main/capture/focused-window.ts b/apps/desktop/src/main/capture/focused-window.ts
index ae47109..905b32a 100644
--- a/apps/desktop/src/main/capture/focused-window.ts
+++ b/apps/desktop/src/main/capture/focused-window.ts
@@ -1,4 +1,6 @@
+import type { ActiveWindow } from '@mimik/capture-native';
import { screen } from 'electron';
+import { loadNative } from './native';
export interface FocusedWindow {
title: string | null;
@@ -27,6 +29,28 @@ export function toDip(bounds: FocusedWindow['bounds']): FocusedWindow['bounds']
}
}
+function fromNative(found: ActiveWindow): FocusedWindowResult {
+ return {
+ ok: true,
+ window: {
+ title: found.title ?? null,
+ app: { name: found.appName, id: found.appPath },
+ bounds: toDip({ x: found.x, y: found.y, width: found.width, height: found.height }),
+ },
+ };
+}
+
+export async function windowAt(point: { x: number; y: number }): Promise<FocusedWindowResult> {
+ if (process.platform === 'win32') {
+ const physical = typeof screen.dipToScreenPoint === 'function' ? screen.dipToScreenPoint(point) : point;
+ const found = await loadNative()
+ .then((native) => native?.windowAt(Math.round(physical.x), Math.round(physical.y)) ?? null)
+ .catch(() => null);
+ if (found) return fromNative(found);
+ }
+ return focusedWindow();
+}
+
export async function focusedWindow(): Promise<FocusedWindowResult> {
if (process.platform === 'linux' && process.env.XDG_SESSION_TYPE === 'wayland') {
return {
@@ -36,6 +60,13 @@ export async function focusedWindow(): Promise<FocusedWindowResult> {
};
}
+ if (process.platform === 'win32') {
+ const found = await loadNative()
+ .then((native) => native?.activeWindow() ?? null)
+ .catch(() => null);
+ if (found) return fromNative(found);
+ }
+
try {
const { activeWindow } = await import('get-windows');
const found = await activeWindow();
diff --git a/apps/desktop/src/main/capture/native.ts b/apps/desktop/src/main/capture/native.ts
new file mode 100644
index 0000000..defc7fc
--- /dev/null
+++ b/apps/desktop/src/main/capture/native.ts
@@ -0,0 +1,25 @@
+import type { ActiveWindow, UiElement } from '@mimik/capture-native';
+
+export type Addon = {
+ elementAtPoint(x: number, y: number): Promise<UiElement | null>;
+ focusedElement(): Promise<UiElement | null>;
+ activeWindow(): ActiveWindow | null;
+ windowAt(x: number, y: number): ActiveWindow | null;
+ keyLabel(keycode: number): string | null;
+ resolveKey(keycode: number, shift: boolean, ctrl: boolean, alt: boolean): string | null;
+ clearDeadKey(): void;
+ isSupported(): boolean;
+};
+
+let addon: Addon | null | undefined;
+
+export async function loadNative(): Promise<Addon | null> {
+ if (addon === undefined) {
+ try {
+ addon = (await import('@mimik/capture-native')) as unknown as Addon;
+ } catch {
+ addon = null;
+ }
+ }
+ return addon;
+}
diff --git a/apps/desktop/src/main/capture/recorder.ts b/apps/desktop/src/main/capture/recorder.ts
index f84db55..e3242d9 100644
--- a/apps/desktop/src/main/capture/recorder.ts
+++ b/apps/desktop/src/main/capture/recorder.ts
@@ -13,7 +13,7 @@ import {
resolveKey,
type ScreenElement,
} from './element';
-import { focusedWindow } from './focused-window';
+import { type FocusedWindowResult, focusedWindow, windowAt } from './focused-window';
import { type InputAction, InputHook, type KeyAction, type PointerAction } from './input-hook';
import type { Region } from './region';
import { type Frame, grabDisplay, type Rect } from './screenshot';
@@ -79,6 +79,7 @@ export interface RecorderHooks {
ignores?: (point: Point) => boolean;
lookup?: (point: Point) => Promise<ScreenElement | null>;
focused?: () => Promise<ScreenElement | null>;
+ windowAt?: (point: Point) => Promise<FocusedWindowResult>;
label?: (keycode: number) => Promise<string | null>;
resolve?: (keycode: number, shift: boolean, ctrl: boolean, alt: boolean) => Promise<string | null>;
reset?: () => Promise<void>;
@@ -112,7 +113,7 @@ export function frameFor(mode: CaptureMode, point: Point, region: Region, window
const display = screen.getDisplayNearestPoint(point).bounds;
if (mode === 'screen') return display;
if (mode === 'window') return window && inside(window, point) ? window : display;
- return inside(region, point) ? region : display;
+ return region;
}
export function isRepeatKey(previous: { keycode: number; at: number } | null, keycode: number, at: number): boolean {
@@ -211,6 +212,7 @@ export class DesktopRecorder {
private readonly ignores: (point: Point) => boolean;
private readonly lookup: (point: Point) => Promise<ScreenElement | null>;
private readonly focused: () => Promise<ScreenElement | null>;
+ private readonly windowAt: (point: Point) => Promise<FocusedWindowResult>;
private readonly label: (keycode: number) => Promise<string | null>;
private readonly resolve: (keycode: number, shift: boolean, ctrl: boolean, alt: boolean) => Promise<string | null>;
private readonly reset: () => Promise<void>;
@@ -226,6 +228,7 @@ export class DesktopRecorder {
this.ignores = hooks.ignores ?? (() => false);
this.lookup = hooks.lookup ?? elementAt;
this.focused = hooks.focused ?? focusedField;
+ this.windowAt = hooks.windowAt ?? windowAt;
this.label = hooks.label ?? keyLabel;
this.resolve = hooks.resolve ?? resolveKey;
this.reset = hooks.reset ?? clearDeadKey;
@@ -283,7 +286,8 @@ export class DesktopRecorder {
this.commitTyping();
const element = this.lookup(point);
const frame = this.shoot(point);
- this.enqueue(() => this.write(action, point, element, frame));
+ const place = this.windowAt(point).catch(() => focusedWindow());
+ this.enqueue(() => this.write(action, point, element, frame, undefined, place));
}
private onKey(action: KeyAction): void {
@@ -361,10 +365,12 @@ export class DesktopRecorder {
}
private shoot(point: Point): Promise<Frame> {
- const { screenshotDelayMs } = this.settings();
+ const { screenshotDelayMs, captureMode } = this.settings();
+ const region = this.region();
+ const at = captureMode === 'region' ? { x: region.x + region.width / 2, y: region.y + region.height / 2 } : point;
const grabbed = this.withHidden(async () => {
await delay(screenshotDelayMs);
- return this.grab(point);
+ return this.grab(at);
});
const frame = Promise.all([grabbed, delay(SETTLE_MS + screenshotDelayMs)]).then(([taken]) => taken);
frame.catch(() => undefined);
@@ -383,7 +389,7 @@ export class DesktopRecorder {
}
async capture(point: Point): Promise<void> {
- await this.write('click', point, this.lookup(point), this.shoot(point));
+ await this.write('click', point, this.lookup(point), this.shoot(point), undefined, this.windowAt(point));
}
async writeTyping(
@@ -409,12 +415,13 @@ export class DesktopRecorder {
element: Promise<ScreenElement | null>,
taken: Promise<Frame>,
inputValue?: string,
+ place?: Promise<FocusedWindowResult>,
): Promise<void> {
const settings = this.settings();
const region = this.region();
const frame = await taken;
- const found = await focusedWindow();
+ const found = await (place ?? focusedWindow());
const framed = frameFor(settings.captureMode, point, region, found.ok ? found.window.bounds : null);
const shot = await frame(framed);
const scale = shot.scaleFactor;