π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
fix(desktop): frame each click by the window or area it belongs to
Luis Ortiz68392d4c4cb6
diff --git a/apps/desktop/scripts/check-pipeline.ts b/apps/desktop/scripts/check-pipeline.tsindex 5d02555..0b2fdfc 100644--- a/apps/desktop/scripts/check-pipeline.ts+++ b/apps/desktop/scripts/check-pipeline.ts@@ -186,8 +186,33 @@ app.whenReady().then(async () => {sameRect(frameFor('window', insidePoint, region, elsewhere), display.bounds) &&sameRect(frameFor('window', insidePoint, region, null), display.bounds) &&sameRect(frameFor('region', insidePoint, region, windowRect), region) &&- sameRect(frameFor('region', outsidePoint, region, windowRect), display.bounds),- detail: 'screen takes the display, window takes the window and falls back twice, region takes the region',+ sameRect(frameFor('region', outsidePoint, region, windowRect), region),+ detail: 'screen takes the display, window takes the window and falls back twice, region always takes the region',+ });++ let framedBy: CaptureRequest | null = null;+ const clickedWindow = { x: region.x + 40, y: region.y + 30, width: 300, height: 200 };+ const windowed = new DesktopRecorder(+ () => region,+ (fn) => fn(),+ (request) => {+ framedBy = request;+ return Promise.resolve(null);+ },+ {+ grab: async () => syntheticDisplay,+ settings: () => ({ ...DEFAULT_CAPTURE_SETTINGS, captureMode: 'window', showCursor: false }),+ lookup: () => Promise.resolve(null),+ windowAt: () =>+ Promise.resolve({ ok: true, window: { title: 'Left pane', app: { name: 'Explorer' }, bounds: clickedWindow } }),+ },+ );+ await windowed.capture({ x: clickedWindow.x + 20, y: clickedWindow.y + 20 });+ const windowShot = (framedBy as CaptureRequest | null)?.image;+ results.push({+ name: 'window mode crops to the window that was clicked',+ ok: windowShot?.width === Math.round(clickedWindow.width * display.scaleFactor),+ detail: `${windowShot?.width ?? 0} px wide for a ${clickedWindow.width} px window`,});const control: ScreenElement = {diff --git a/apps/desktop/src/main/capture/element.ts b/apps/desktop/src/main/capture/element.tsindex 494e34f..6e6e78c 100644--- a/apps/desktop/src/main/capture/element.ts+++ b/apps/desktop/src/main/capture/element.ts@@ -1,6 +1,7 @@import type { ElementNode, UiElement } from '@mimik/capture-native';import { screen } from 'electron';import { toDip } from './focused-window';+import { loadNative as load } from './native';const LOOKUP_TIMEOUT_MS = 1500;@@ -18,28 +19,6 @@ export interface ScreenElement {const TEXT_ROLES = new Set(['textbox', 'combobox', 'document']);-type Addon = {- elementAtPoint(x: number, y: number): Promise<UiElement | null>;- focusedElement(): Promise<UiElement | null>;- keyLabel(keycode: number): string | null;- resolveKey(keycode: number, shift: boolean, ctrl: boolean, alt: boolean): string | null;- clearDeadKey(): void;- isSupported(): boolean;-};--let addon: Addon | null | undefined;--async function load(): Promise<Addon | null> {- if (addon === undefined) {- try {- addon = (await import('@mimik/capture-native')) as unknown as Addon;- } catch {- addon = null;- }- }- return addon;-}-function within<T>(work: Promise<T>, ms: number): Promise<T | null> {return new Promise((resolve) => {const timer = setTimeout(() => resolve(null), ms);diff --git a/apps/desktop/src/main/capture/focused-window.ts b/apps/desktop/src/main/capture/focused-window.tsindex ae47109..905b32a 100644--- a/apps/desktop/src/main/capture/focused-window.ts+++ b/apps/desktop/src/main/capture/focused-window.ts@@ -1,4 +1,6 @@+import type { ActiveWindow } from '@mimik/capture-native';import { screen } from 'electron';+import { loadNative } from './native';export interface FocusedWindow {title: string | null;@@ -27,6 +29,28 @@ export function toDip(bounds: FocusedWindow['bounds']): FocusedWindow['bounds']}}+function fromNative(found: ActiveWindow): FocusedWindowResult {+ return {+ ok: true,+ window: {+ title: found.title ?? null,+ app: { name: found.appName, id: found.appPath },+ bounds: toDip({ x: found.x, y: found.y, width: found.width, height: found.height }),+ },+ };+}++export async function windowAt(point: { x: number; y: number }): Promise<FocusedWindowResult> {+ if (process.platform === 'win32') {+ const physical = typeof screen.dipToScreenPoint === 'function' ? screen.dipToScreenPoint(point) : point;+ const found = await loadNative()+ .then((native) => native?.windowAt(Math.round(physical.x), Math.round(physical.y)) ?? null)+ .catch(() => null);+ if (found) return fromNative(found);+ }+ return focusedWindow();+}+export async function focusedWindow(): Promise<FocusedWindowResult> {if (process.platform === 'linux' && process.env.XDG_SESSION_TYPE === 'wayland') {return {@@ -36,6 +60,13 @@ export async function focusedWindow(): Promise<FocusedWindowResult> {};}+ if (process.platform === 'win32') {+ const found = await loadNative()+ .then((native) => native?.activeWindow() ?? null)+ .catch(() => null);+ if (found) return fromNative(found);+ }+try {const { activeWindow } = await import('get-windows');const found = await activeWindow();diff --git a/apps/desktop/src/main/capture/native.ts b/apps/desktop/src/main/capture/native.tsnew file mode 100644index 0000000..defc7fc--- /dev/null+++ b/apps/desktop/src/main/capture/native.ts@@ -0,0 +1,25 @@+import type { ActiveWindow, UiElement } from '@mimik/capture-native';++export type Addon = {+ elementAtPoint(x: number, y: number): Promise<UiElement | null>;+ focusedElement(): Promise<UiElement | null>;+ activeWindow(): ActiveWindow | null;+ windowAt(x: number, y: number): ActiveWindow | null;+ keyLabel(keycode: number): string | null;+ resolveKey(keycode: number, shift: boolean, ctrl: boolean, alt: boolean): string | null;+ clearDeadKey(): void;+ isSupported(): boolean;+};++let addon: Addon | null | undefined;++export async function loadNative(): Promise<Addon | null> {+ if (addon === undefined) {+ try {+ addon = (await import('@mimik/capture-native')) as unknown as Addon;+ } catch {+ addon = null;+ }+ }+ return addon;+}diff --git a/apps/desktop/src/main/capture/recorder.ts b/apps/desktop/src/main/capture/recorder.tsindex f84db55..e3242d9 100644--- a/apps/desktop/src/main/capture/recorder.ts+++ b/apps/desktop/src/main/capture/recorder.ts@@ -13,7 +13,7 @@ import {resolveKey,type ScreenElement,} from './element';-import { focusedWindow } from './focused-window';+import { type FocusedWindowResult, focusedWindow, windowAt } from './focused-window';import { type InputAction, InputHook, type KeyAction, type PointerAction } from './input-hook';import type { Region } from './region';import { type Frame, grabDisplay, type Rect } from './screenshot';@@ -79,6 +79,7 @@ export interface RecorderHooks {ignores?: (point: Point) => boolean;lookup?: (point: Point) => Promise<ScreenElement | null>;focused?: () => Promise<ScreenElement | null>;+ windowAt?: (point: Point) => Promise<FocusedWindowResult>;label?: (keycode: number) => Promise<string | null>;resolve?: (keycode: number, shift: boolean, ctrl: boolean, alt: boolean) => Promise<string | null>;reset?: () => Promise<void>;@@ -112,7 +113,7 @@ export function frameFor(mode: CaptureMode, point: Point, region: Region, windowconst display = screen.getDisplayNearestPoint(point).bounds;if (mode === 'screen') return display;if (mode === 'window') return window && inside(window, point) ? window : display;- return inside(region, point) ? region : display;+ return region;}export function isRepeatKey(previous: { keycode: number; at: number } | null, keycode: number, at: number): boolean {@@ -211,6 +212,7 @@ export class DesktopRecorder {private readonly ignores: (point: Point) => boolean;private readonly lookup: (point: Point) => Promise<ScreenElement | null>;private readonly focused: () => Promise<ScreenElement | null>;+ private readonly windowAt: (point: Point) => Promise<FocusedWindowResult>;private readonly label: (keycode: number) => Promise<string | null>;private readonly resolve: (keycode: number, shift: boolean, ctrl: boolean, alt: boolean) => Promise<string | null>;private readonly reset: () => Promise<void>;@@ -226,6 +228,7 @@ export class DesktopRecorder {this.ignores = hooks.ignores ?? (() => false);this.lookup = hooks.lookup ?? elementAt;this.focused = hooks.focused ?? focusedField;+ this.windowAt = hooks.windowAt ?? windowAt;this.label = hooks.label ?? keyLabel;this.resolve = hooks.resolve ?? resolveKey;this.reset = hooks.reset ?? clearDeadKey;@@ -283,7 +286,8 @@ export class DesktopRecorder {this.commitTyping();const element = this.lookup(point);const frame = this.shoot(point);- this.enqueue(() => this.write(action, point, element, frame));+ const place = this.windowAt(point).catch(() => focusedWindow());+ this.enqueue(() => this.write(action, point, element, frame, undefined, place));}private onKey(action: KeyAction): void {@@ -361,10 +365,12 @@ export class DesktopRecorder {}private shoot(point: Point): Promise<Frame> {- const { screenshotDelayMs } = this.settings();+ const { screenshotDelayMs, captureMode } = this.settings();+ const region = this.region();+ const at = captureMode === 'region' ? { x: region.x + region.width / 2, y: region.y + region.height / 2 } : point;const grabbed = this.withHidden(async () => {await delay(screenshotDelayMs);- return this.grab(point);+ return this.grab(at);});const frame = Promise.all([grabbed, delay(SETTLE_MS + screenshotDelayMs)]).then(([taken]) => taken);frame.catch(() => undefined);@@ -383,7 +389,7 @@ export class DesktopRecorder {}async capture(point: Point): Promise<void> {- await this.write('click', point, this.lookup(point), this.shoot(point));+ await this.write('click', point, this.lookup(point), this.shoot(point), undefined, this.windowAt(point));}async writeTyping(@@ -409,12 +415,13 @@ export class DesktopRecorder {element: Promise<ScreenElement | null>,taken: Promise<Frame>,inputValue?: string,+ place?: Promise<FocusedWindowResult>,): Promise<void> {const settings = this.settings();const region = this.region();const frame = await taken;- const found = await focusedWindow();+ const found = await (place ?? focusedWindow());const framed = frameFor(settings.captureMode, point, region, found.ok ? found.window.bounds : null);const shot = await frame(framed);const scale = shot.scaleFactor;