π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
feat(desktop): record clicks in the capture region into a guide
Luis Ortiz8eec72598faf
diff --git a/apps/desktop/src/env.d.ts b/apps/desktop/src/env.d.tsnew file mode 100644index 0000000..e815d50--- /dev/null+++ b/apps/desktop/src/env.d.ts@@ -0,0 +1,13 @@+import type { MimikDesktopApi } from './preload/index';+import type { MimikOverlayApi } from './preload/overlay';++declare global {+ interface ImportMetaEnv {+ readonly COMMAND?: string;+ }++ interface Window {+ mimik: MimikDesktopApi;+ mimikOverlay: MimikOverlayApi;+ }+}diff --git a/apps/desktop/src/main/ask.ts b/apps/desktop/src/main/ask.tsnew file mode 100644index 0000000..dfcb4de--- /dev/null+++ b/apps/desktop/src/main/ask.ts@@ -0,0 +1,22 @@+import { ipcMain, type WebContents } from 'electron';++let seq = 0;++export function ask<T>(target: WebContents | null, channel: string, payload?: unknown, timeoutMs = 15_000): Promise<T> {+ if (!target || target.isDestroyed()) return Promise.reject(new Error(`${channel}: no renderer available`));++ const replyChannel = `${channel}:reply:${++seq}`;+ return new Promise<T>((resolve, reject) => {+ const timer = setTimeout(() => {+ ipcMain.removeAllListeners(replyChannel);+ reject(new Error(`${channel}: renderer did not reply within ${timeoutMs}ms`));+ }, timeoutMs);++ ipcMain.once(replyChannel, (_event, result: T) => {+ clearTimeout(timer);+ resolve(result);+ });++ target.send(channel, replyChannel, payload);+ });+}diff --git a/apps/desktop/src/main/capture/recorder.ts b/apps/desktop/src/main/capture/recorder.tsnew file mode 100644index 0000000..6558d95--- /dev/null+++ b/apps/desktop/src/main/capture/recorder.ts@@ -0,0 +1,132 @@+import type { CaptureImage } from '@mimik/core/capture/sink';+import type { ElementMeta } from '@mimik/core/guides/types';+import { nativeImage, screen } from 'electron';+import { focusedWindow } from './focused-window';+import { type InputAction, InputHook } from './input-hook';+import type { Region } from './region';+import { type Capture, captureDisplay } from './screenshot';++const TARGET_SIZE = 28;+const SETTLE_MS = 60;++export interface CaptureRequest {+ action: string;+ elementMeta: ElementMeta;+ image: CaptureImage;+}++export type RecorderStart = { ok: true } | { ok: false; reason: string; detail: string };++function delay(ms: number): Promise<void> {+ return new Promise((resolve) => setTimeout(resolve, ms));+}++function inside(region: Region, point: { x: number; y: number }): boolean {+ return (+ point.x >= region.x &&+ point.y >= region.y &&+ point.x < region.x + region.width &&+ point.y < region.y + region.height+ );+}++export class DesktopRecorder {+ private hook = new InputHook();+ private queue: Promise<unknown> = Promise.resolve();+ private paused = false;+ private running = false;++ constructor(+ private readonly region: () => Region,+ private readonly withHidden: <T>(fn: () => Promise<T>) => Promise<T>,+ private readonly send: (request: CaptureRequest) => Promise<unknown>,+ private readonly grab: (displayId: number) => Promise<Capture> = captureDisplay,+ ) {}++ async start(): Promise<RecorderStart> {+ if (this.running) return { ok: true };+ const started = await this.hook.start((action) => this.onAction(action));+ if (!started.ok) return { ok: false, reason: started.reason, detail: started.detail };+ this.running = true;+ this.paused = false;+ return { ok: true };+ }++ pause(): void {+ this.paused = true;+ }++ resume(): void {+ this.paused = false;+ }++ stop(): void {+ this.hook.stop();+ this.running = false;+ this.paused = false;+ }++ get isRecording(): boolean {+ return this.running && !this.paused;+ }++ private onAction(action: InputAction): void {+ if (action.kind !== 'click' || !this.isRecording) return;+ const point = { x: action.x, y: action.y };+ if (!inside(this.region(), point)) return;+ this.enqueue(point);+ }++ private enqueue(point: { x: number; y: number }): void {+ this.queue = this.queue.then(() => this.capture(point)).catch(() => undefined);+ }++ async capture(point: { x: number; y: number }): Promise<void> {+ const region = this.region();+ const display = screen.getDisplayMatching(region);+ const scale = display.scaleFactor;++ const shot = await this.withHidden(async () => {+ await delay(SETTLE_MS);+ return this.grab(display.id);+ });++ const cropped = nativeImage.createFromBuffer(Buffer.from(shot.png)).crop({+ x: Math.round((region.x - display.bounds.x) * scale),+ y: Math.round((region.y - display.bounds.y) * scale),+ width: Math.round(region.width * scale),+ height: Math.round(region.height * scale),+ });+ const size = cropped.getSize();++ const local = { x: point.x - region.x, y: point.y - region.y };+ const found = await focusedWindow();++ await this.send({+ action: 'click',+ elementMeta: {+ source: 'screen',+ textContent: null,+ ariaLabel: null,+ placeholder: null,+ altText: null,+ name: null,+ role: null,+ rect: {+ x: local.x - TARGET_SIZE / 2,+ y: local.y - TARGET_SIZE / 2,+ width: TARGET_SIZE,+ height: TARGET_SIZE,+ },+ devicePixelRatio: scale,+ clickPoint: local,+ ...(found.ok ? { app: found.window.app, window: { title: found.window.title } } : {}),+ },+ image: { png: cropped.toPNG(), width: size.width, height: size.height },+ });+ }++ async drain(): Promise<void> {+ await this.queue;+ }+}diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.tsindex e1f5b44..916fa39 100644--- a/apps/desktop/src/main/index.ts+++ b/apps/desktop/src/main/index.ts@@ -1,11 +1,15 @@import { join } from 'node:path';-import { app, BrowserWindow, ipcMain, Menu, nativeImage, shell, Tray } from 'electron';+import { app, BrowserWindow, dialog, ipcMain, Menu, nativeImage, shell, Tray } from 'electron';+import { ask } from './ask';+import { DesktopRecorder } from './capture/recorder';import { CaptureOverlay, type OverlayCommand } from './overlay';import { checkForUpdates } from './updater';let mainWindow: BrowserWindow | null = null;let tray: Tray | null = null;let overlay: CaptureOverlay | null = null;+let recorder: DesktopRecorder | null = null;+let guideId: string | null = null;function resource(file: string): string {return app.isPackaged ? join(process.resourcesPath, file) : join(__dirname, '../../resources', file);@@ -102,7 +106,29 @@ function createTray(): void {}function broadcastOverlay(command: OverlayCommand): void {- mainWindow?.webContents.send('mimik:capture:command', command, overlay?.state, overlay?.region);+ mainWindow?.webContents.send('mimik:capture:command', command, overlay?.state, overlay?.region, guideId);+}++async function onOverlayCommand(command: OverlayCommand): Promise<void> {+ if (command === 'start' && !guideId) {+ guideId = await ask<string>(mainWindow?.webContents ?? null, 'mimik:capture:startGuide');+ const started = await recorder?.start();+ if (started && !started.ok) {+ guideId = null;+ overlay?.hide();+ dialog.showErrorBox('Mimik cannot record', started.detail);+ return;+ }+ } else if (command === 'pause') {+ recorder?.pause();+ } else if (command === 'resume') {+ recorder?.resume();+ } else if (command === 'stop' || command === 'cancel') {+ recorder?.stop();+ await recorder?.drain();+ guideId = null;+ }+ broadcastOverlay(command);}let isQuitting = false;@@ -125,7 +151,13 @@ if (!app.requestSingleInstanceLock()) {});ipcMain.handle('mimik:version', () => app.getVersion());- overlay = new CaptureOverlay(broadcastOverlay);+ overlay = new CaptureOverlay((command) => void onOverlayCommand(command));+ recorder = new DesktopRecorder(+ () => overlay?.region ?? { x: 0, y: 0, width: 0, height: 0 },+ (fn) => (overlay ? overlay.withHidden(fn) : fn()),+ (request) =>+ ask(mainWindow?.webContents ?? null, 'mimik:capture:step', { ...request, guideId }).catch(() => undefined),+ );ipcMain.handle('mimik:capture:region', () => overlay?.region);ipcMain.handle('mimik:capture:edit', () => overlay?.edit());@@ -137,6 +169,7 @@ if (!app.requestSingleInstanceLock()) {app.on('activate', () => showWindow());app.on('before-quit', () => {isQuitting = true;+ recorder?.stop();overlay?.destroy();});app.on('window-all-closed', () => {});diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.tsindex 4c2b23b..b8bafde 100644--- a/apps/desktop/src/preload/index.ts+++ b/apps/desktop/src/preload/index.ts@@ -18,6 +18,15 @@ const api = {);},},+ onRequest: (channel: string, handler: (payload: unknown) => Promise<unknown>): void => {+ ipcRenderer.on(channel, async (_event, replyChannel: string, payload: unknown) => {+ try {+ ipcRenderer.send(replyChannel, await handler(payload));+ } catch (error) {+ ipcRenderer.send(replyChannel, { error: error instanceof Error ? error.message : String(error) });+ }+ });+ },openAtLogin: {get: (): Promise<boolean> => ipcRenderer.invoke('mimik:openAtLogin:get'),set: (enabled: boolean): Promise<boolean> => ipcRenderer.invoke('mimik:openAtLogin:set', enabled),diff --git a/apps/desktop/src/renderer/capture-sink.ts b/apps/desktop/src/renderer/capture-sink.tsnew file mode 100644index 0000000..e23cf1a--- /dev/null+++ b/apps/desktop/src/renderer/capture-sink.ts@@ -0,0 +1,90 @@+import type {+ CaptureSink,+ CaptureStepData,+ CaptureStepResponse,+ FinalizeInputStepData,+ FinalizeInputStepResponse,+ UpdateInputStepData,+ UpdateInputStepResponse,+} from '@mimik/core/capture/sink';+import { buildFallbackDescription } from '@mimik/core/capture/step-description';+import { localStorage } from '@mimik/core/env';+import {+ addStepToGuide,+ createGuide,+ createStep,+ getStepsForGuide,+ saveScreenshot,+ updateStepDescription,+} from '@mimik/core/guides/service';+import type { Screenshot } from '@mimik/core/guides/types';+import { DEFAULT_TARGET_COLOR } from '@mimik/core/screenshot/types';++export class DesktopCaptureSink implements CaptureSink {+ async startGuide(): Promise<string> {+ const guide = await createGuide(crypto.randomUUID());+ return guide.id;+ }++ async captureStep(data: CaptureStepData): Promise<CaptureStepResponse> {+ const stepId = crypto.randomUUID();+ const meta = data.elementMeta;+ const index = (await getStepsForGuide(data.guideId)).length;++ let screenshotId: string | undefined;+ if (data.image) {+ const { targetColor } = await localStorage.get(['targetColor']);+ const ratio = meta.devicePixelRatio;+ const screenshot: Screenshot = {+ id: crypto.randomUUID(),+ stepId,+ blob: new Blob([Uint8Array.from(data.image.png)], { type: 'image/png' }),+ mimeType: 'image/png',+ width: data.image.width,+ height: data.image.height,+ bounds: { x: meta.rect.x, y: meta.rect.y, width: meta.rect.width, height: meta.rect.height },+ pixelRatio: ratio,+ clickPoint: meta.clickPoint,+ edits: {+ target: {+ x: meta.rect.x * ratio,+ y: meta.rect.y * ratio,+ width: meta.rect.width * ratio,+ height: meta.rect.height * ratio,+ border: 'dashed',+ color: targetColor || DEFAULT_TARGET_COLOR,+ },+ },+ };+ await saveScreenshot(screenshot);+ screenshotId = screenshot.id;+ }++ await createStep({+ id: stepId,+ guideId: data.guideId,+ index,+ description: buildFallbackDescription(data.action, meta),+ action: data.action,+ url: '',+ app: meta.app,+ window: meta.window,+ timestamp: Date.now(),+ screenshotId,+ elementMeta: meta,+ descriptionSource: 'heuristic',+ });+ await addStepToGuide(data.guideId, stepId);++ return { stepId };+ }++ async updateInputStep(data: UpdateInputStepData): Promise<UpdateInputStepResponse> {+ await updateStepDescription(data.stepId, data.description);+ return { updated: true };+ }++ async finalizeInputStep(_data: FinalizeInputStepData): Promise<FinalizeInputStepResponse> {+ return { updated: false };+ }+}diff --git a/apps/desktop/src/renderer/main.ts b/apps/desktop/src/renderer/main.tsindex 384eea1..ecd0b90 100644--- a/apps/desktop/src/renderer/main.ts+++ b/apps/desktop/src/renderer/main.ts@@ -1,26 +1,13 @@import './core-env';import { PRESET_LABELS } from '@mimik/core/blur/regexes';+import type { CaptureStepData } from '@mimik/core/capture/sink';+import { getGuides } from '@mimik/core/guides/service';+import { DesktopCaptureSink } from './capture-sink';import './style.css';-declare global {- interface Window {- mimik: {- version(): Promise<string>;- openAtLogin: { get(): Promise<boolean>; set(enabled: boolean): Promise<boolean> };- capture: {- region(): Promise<{ x: number; y: number; width: number; height: number }>;- edit(): Promise<void>;- onCommand(- handler: (- command: string,- state: string,- region: { x: number; y: number; width: number; height: number },- ) => void,- ): void;- };- };- }-}+const sink = new DesktopCaptureSink();+window.mimik.onRequest('mimik:capture:startGuide', () => sink.startGuide());+window.mimik.onRequest('mimik:capture:step', (payload) => sink.captureStep(payload as CaptureStepData));const root = document.getElementById('root') as HTMLElement;@@ -60,11 +47,20 @@ async function render(): Promise<void> {edit.textContent = 'Set capture area';edit.addEventListener('click', () => window.mimik.capture.edit());+ const library = document.createElement('p');+ library.className = 'meta';+ const refresh = async () => {+ const guides = await getGuides();+ library.textContent = `${guides.length} guide(s) in the library`;+ };+ await refresh();+window.mimik.capture.onCommand((_command, _state, next) => {area.textContent = `Capture area ${next.width} Γ ${next.height} at ${next.x}, ${next.y}`;+ void refresh();});- root.append(title, meta, label, area, edit, shared);+ root.append(title, meta, label, area, edit, library, shared);}render();