Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

feat(desktop): record clicks in the capture region into a guide

Luis Ortiz8eec72598faf
diff --git a/apps/desktop/src/env.d.ts b/apps/desktop/src/env.d.ts
new file mode 100644
index 0000000..e815d50
--- /dev/null
+++ b/apps/desktop/src/env.d.ts
@@ -0,0 +1,13 @@
+import type { MimikDesktopApi } from './preload/index';
+import type { MimikOverlayApi } from './preload/overlay';
+
+declare global {
+ interface ImportMetaEnv {
+ readonly COMMAND?: string;
+ }
+
+ interface Window {
+ mimik: MimikDesktopApi;
+ mimikOverlay: MimikOverlayApi;
+ }
+}
diff --git a/apps/desktop/src/main/ask.ts b/apps/desktop/src/main/ask.ts
new file mode 100644
index 0000000..dfcb4de
--- /dev/null
+++ b/apps/desktop/src/main/ask.ts
@@ -0,0 +1,22 @@
+import { ipcMain, type WebContents } from 'electron';
+
+let seq = 0;
+
+export function ask<T>(target: WebContents | null, channel: string, payload?: unknown, timeoutMs = 15_000): Promise<T> {
+ if (!target || target.isDestroyed()) return Promise.reject(new Error(`${channel}: no renderer available`));
+
+ const replyChannel = `${channel}:reply:${++seq}`;
+ return new Promise<T>((resolve, reject) => {
+ const timer = setTimeout(() => {
+ ipcMain.removeAllListeners(replyChannel);
+ reject(new Error(`${channel}: renderer did not reply within ${timeoutMs}ms`));
+ }, timeoutMs);
+
+ ipcMain.once(replyChannel, (_event, result: T) => {
+ clearTimeout(timer);
+ resolve(result);
+ });
+
+ target.send(channel, replyChannel, payload);
+ });
+}
diff --git a/apps/desktop/src/main/capture/recorder.ts b/apps/desktop/src/main/capture/recorder.ts
new file mode 100644
index 0000000..6558d95
--- /dev/null
+++ b/apps/desktop/src/main/capture/recorder.ts
@@ -0,0 +1,132 @@
+import type { CaptureImage } from '@mimik/core/capture/sink';
+import type { ElementMeta } from '@mimik/core/guides/types';
+import { nativeImage, screen } from 'electron';
+import { focusedWindow } from './focused-window';
+import { type InputAction, InputHook } from './input-hook';
+import type { Region } from './region';
+import { type Capture, captureDisplay } from './screenshot';
+
+const TARGET_SIZE = 28;
+const SETTLE_MS = 60;
+
+export interface CaptureRequest {
+ action: string;
+ elementMeta: ElementMeta;
+ image: CaptureImage;
+}
+
+export type RecorderStart = { ok: true } | { ok: false; reason: string; detail: string };
+
+function delay(ms: number): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function inside(region: Region, point: { x: number; y: number }): boolean {
+ return (
+ point.x >= region.x &&
+ point.y >= region.y &&
+ point.x < region.x + region.width &&
+ point.y < region.y + region.height
+ );
+}
+
+export class DesktopRecorder {
+ private hook = new InputHook();
+ private queue: Promise<unknown> = Promise.resolve();
+ private paused = false;
+ private running = false;
+
+ constructor(
+ private readonly region: () => Region,
+ private readonly withHidden: <T>(fn: () => Promise<T>) => Promise<T>,
+ private readonly send: (request: CaptureRequest) => Promise<unknown>,
+ private readonly grab: (displayId: number) => Promise<Capture> = captureDisplay,
+ ) {}
+
+ async start(): Promise<RecorderStart> {
+ if (this.running) return { ok: true };
+ const started = await this.hook.start((action) => this.onAction(action));
+ if (!started.ok) return { ok: false, reason: started.reason, detail: started.detail };
+ this.running = true;
+ this.paused = false;
+ return { ok: true };
+ }
+
+ pause(): void {
+ this.paused = true;
+ }
+
+ resume(): void {
+ this.paused = false;
+ }
+
+ stop(): void {
+ this.hook.stop();
+ this.running = false;
+ this.paused = false;
+ }
+
+ get isRecording(): boolean {
+ return this.running && !this.paused;
+ }
+
+ private onAction(action: InputAction): void {
+ if (action.kind !== 'click' || !this.isRecording) return;
+ const point = { x: action.x, y: action.y };
+ if (!inside(this.region(), point)) return;
+ this.enqueue(point);
+ }
+
+ private enqueue(point: { x: number; y: number }): void {
+ this.queue = this.queue.then(() => this.capture(point)).catch(() => undefined);
+ }
+
+ async capture(point: { x: number; y: number }): Promise<void> {
+ const region = this.region();
+ const display = screen.getDisplayMatching(region);
+ const scale = display.scaleFactor;
+
+ const shot = await this.withHidden(async () => {
+ await delay(SETTLE_MS);
+ return this.grab(display.id);
+ });
+
+ const cropped = nativeImage.createFromBuffer(Buffer.from(shot.png)).crop({
+ x: Math.round((region.x - display.bounds.x) * scale),
+ y: Math.round((region.y - display.bounds.y) * scale),
+ width: Math.round(region.width * scale),
+ height: Math.round(region.height * scale),
+ });
+ const size = cropped.getSize();
+
+ const local = { x: point.x - region.x, y: point.y - region.y };
+ const found = await focusedWindow();
+
+ await this.send({
+ action: 'click',
+ elementMeta: {
+ source: 'screen',
+ textContent: null,
+ ariaLabel: null,
+ placeholder: null,
+ altText: null,
+ name: null,
+ role: null,
+ rect: {
+ x: local.x - TARGET_SIZE / 2,
+ y: local.y - TARGET_SIZE / 2,
+ width: TARGET_SIZE,
+ height: TARGET_SIZE,
+ },
+ devicePixelRatio: scale,
+ clickPoint: local,
+ ...(found.ok ? { app: found.window.app, window: { title: found.window.title } } : {}),
+ },
+ image: { png: cropped.toPNG(), width: size.width, height: size.height },
+ });
+ }
+
+ async drain(): Promise<void> {
+ await this.queue;
+ }
+}
diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts
index e1f5b44..916fa39 100644
--- a/apps/desktop/src/main/index.ts
+++ b/apps/desktop/src/main/index.ts
@@ -1,11 +1,15 @@
import { join } from 'node:path';
-import { app, BrowserWindow, ipcMain, Menu, nativeImage, shell, Tray } from 'electron';
+import { app, BrowserWindow, dialog, ipcMain, Menu, nativeImage, shell, Tray } from 'electron';
+import { ask } from './ask';
+import { DesktopRecorder } from './capture/recorder';
import { CaptureOverlay, type OverlayCommand } from './overlay';
import { checkForUpdates } from './updater';
let mainWindow: BrowserWindow | null = null;
let tray: Tray | null = null;
let overlay: CaptureOverlay | null = null;
+let recorder: DesktopRecorder | null = null;
+let guideId: string | null = null;
function resource(file: string): string {
return app.isPackaged ? join(process.resourcesPath, file) : join(__dirname, '../../resources', file);
@@ -102,7 +106,29 @@ function createTray(): void {
}
function broadcastOverlay(command: OverlayCommand): void {
- mainWindow?.webContents.send('mimik:capture:command', command, overlay?.state, overlay?.region);
+ mainWindow?.webContents.send('mimik:capture:command', command, overlay?.state, overlay?.region, guideId);
+}
+
+async function onOverlayCommand(command: OverlayCommand): Promise<void> {
+ if (command === 'start' && !guideId) {
+ guideId = await ask<string>(mainWindow?.webContents ?? null, 'mimik:capture:startGuide');
+ const started = await recorder?.start();
+ if (started && !started.ok) {
+ guideId = null;
+ overlay?.hide();
+ dialog.showErrorBox('Mimik cannot record', started.detail);
+ return;
+ }
+ } else if (command === 'pause') {
+ recorder?.pause();
+ } else if (command === 'resume') {
+ recorder?.resume();
+ } else if (command === 'stop' || command === 'cancel') {
+ recorder?.stop();
+ await recorder?.drain();
+ guideId = null;
+ }
+ broadcastOverlay(command);
}
let isQuitting = false;
@@ -125,7 +151,13 @@ if (!app.requestSingleInstanceLock()) {
});
ipcMain.handle('mimik:version', () => app.getVersion());
- overlay = new CaptureOverlay(broadcastOverlay);
+ overlay = new CaptureOverlay((command) => void onOverlayCommand(command));
+ recorder = new DesktopRecorder(
+ () => overlay?.region ?? { x: 0, y: 0, width: 0, height: 0 },
+ (fn) => (overlay ? overlay.withHidden(fn) : fn()),
+ (request) =>
+ ask(mainWindow?.webContents ?? null, 'mimik:capture:step', { ...request, guideId }).catch(() => undefined),
+ );
ipcMain.handle('mimik:capture:region', () => overlay?.region);
ipcMain.handle('mimik:capture:edit', () => overlay?.edit());
@@ -137,6 +169,7 @@ if (!app.requestSingleInstanceLock()) {
app.on('activate', () => showWindow());
app.on('before-quit', () => {
isQuitting = true;
+ recorder?.stop();
overlay?.destroy();
});
app.on('window-all-closed', () => {});
diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts
index 4c2b23b..b8bafde 100644
--- a/apps/desktop/src/preload/index.ts
+++ b/apps/desktop/src/preload/index.ts
@@ -18,6 +18,15 @@ const api = {
);
},
},
+ onRequest: (channel: string, handler: (payload: unknown) => Promise<unknown>): void => {
+ ipcRenderer.on(channel, async (_event, replyChannel: string, payload: unknown) => {
+ try {
+ ipcRenderer.send(replyChannel, await handler(payload));
+ } catch (error) {
+ ipcRenderer.send(replyChannel, { error: error instanceof Error ? error.message : String(error) });
+ }
+ });
+ },
openAtLogin: {
get: (): Promise<boolean> => ipcRenderer.invoke('mimik:openAtLogin:get'),
set: (enabled: boolean): Promise<boolean> => ipcRenderer.invoke('mimik:openAtLogin:set', enabled),
diff --git a/apps/desktop/src/renderer/capture-sink.ts b/apps/desktop/src/renderer/capture-sink.ts
new file mode 100644
index 0000000..e23cf1a
--- /dev/null
+++ b/apps/desktop/src/renderer/capture-sink.ts
@@ -0,0 +1,90 @@
+import type {
+ CaptureSink,
+ CaptureStepData,
+ CaptureStepResponse,
+ FinalizeInputStepData,
+ FinalizeInputStepResponse,
+ UpdateInputStepData,
+ UpdateInputStepResponse,
+} from '@mimik/core/capture/sink';
+import { buildFallbackDescription } from '@mimik/core/capture/step-description';
+import { localStorage } from '@mimik/core/env';
+import {
+ addStepToGuide,
+ createGuide,
+ createStep,
+ getStepsForGuide,
+ saveScreenshot,
+ updateStepDescription,
+} from '@mimik/core/guides/service';
+import type { Screenshot } from '@mimik/core/guides/types';
+import { DEFAULT_TARGET_COLOR } from '@mimik/core/screenshot/types';
+
+export class DesktopCaptureSink implements CaptureSink {
+ async startGuide(): Promise<string> {
+ const guide = await createGuide(crypto.randomUUID());
+ return guide.id;
+ }
+
+ async captureStep(data: CaptureStepData): Promise<CaptureStepResponse> {
+ const stepId = crypto.randomUUID();
+ const meta = data.elementMeta;
+ const index = (await getStepsForGuide(data.guideId)).length;
+
+ let screenshotId: string | undefined;
+ if (data.image) {
+ const { targetColor } = await localStorage.get(['targetColor']);
+ const ratio = meta.devicePixelRatio;
+ const screenshot: Screenshot = {
+ id: crypto.randomUUID(),
+ stepId,
+ blob: new Blob([Uint8Array.from(data.image.png)], { type: 'image/png' }),
+ mimeType: 'image/png',
+ width: data.image.width,
+ height: data.image.height,
+ bounds: { x: meta.rect.x, y: meta.rect.y, width: meta.rect.width, height: meta.rect.height },
+ pixelRatio: ratio,
+ clickPoint: meta.clickPoint,
+ edits: {
+ target: {
+ x: meta.rect.x * ratio,
+ y: meta.rect.y * ratio,
+ width: meta.rect.width * ratio,
+ height: meta.rect.height * ratio,
+ border: 'dashed',
+ color: targetColor || DEFAULT_TARGET_COLOR,
+ },
+ },
+ };
+ await saveScreenshot(screenshot);
+ screenshotId = screenshot.id;
+ }
+
+ await createStep({
+ id: stepId,
+ guideId: data.guideId,
+ index,
+ description: buildFallbackDescription(data.action, meta),
+ action: data.action,
+ url: '',
+ app: meta.app,
+ window: meta.window,
+ timestamp: Date.now(),
+ screenshotId,
+ elementMeta: meta,
+ descriptionSource: 'heuristic',
+ });
+ await addStepToGuide(data.guideId, stepId);
+
+ return { stepId };
+ }
+
+ async updateInputStep(data: UpdateInputStepData): Promise<UpdateInputStepResponse> {
+ await updateStepDescription(data.stepId, data.description);
+ return { updated: true };
+ }
+
+ async finalizeInputStep(_data: FinalizeInputStepData): Promise<FinalizeInputStepResponse> {
+ return { updated: false };
+ }
+}
diff --git a/apps/desktop/src/renderer/main.ts b/apps/desktop/src/renderer/main.ts
index 384eea1..ecd0b90 100644
--- a/apps/desktop/src/renderer/main.ts
+++ b/apps/desktop/src/renderer/main.ts
@@ -1,26 +1,13 @@
import './core-env';
import { PRESET_LABELS } from '@mimik/core/blur/regexes';
+import type { CaptureStepData } from '@mimik/core/capture/sink';
+import { getGuides } from '@mimik/core/guides/service';
+import { DesktopCaptureSink } from './capture-sink';
import './style.css';
-declare global {
- interface Window {
- mimik: {
- version(): Promise<string>;
- openAtLogin: { get(): Promise<boolean>; set(enabled: boolean): Promise<boolean> };
- capture: {
- region(): Promise<{ x: number; y: number; width: number; height: number }>;
- edit(): Promise<void>;
- onCommand(
- handler: (
- command: string,
- state: string,
- region: { x: number; y: number; width: number; height: number },
- ) => void,
- ): void;
- };
- };
- }
-}
+const sink = new DesktopCaptureSink();
+window.mimik.onRequest('mimik:capture:startGuide', () => sink.startGuide());
+window.mimik.onRequest('mimik:capture:step', (payload) => sink.captureStep(payload as CaptureStepData));
const root = document.getElementById('root') as HTMLElement;
@@ -60,11 +47,20 @@ async function render(): Promise<void> {
edit.textContent = 'Set capture area';
edit.addEventListener('click', () => window.mimik.capture.edit());
+ const library = document.createElement('p');
+ library.className = 'meta';
+ const refresh = async () => {
+ const guides = await getGuides();
+ library.textContent = `${guides.length} guide(s) in the library`;
+ };
+ await refresh();
+
window.mimik.capture.onCommand((_command, _state, next) => {
area.textContent = `Capture area ${next.width} Γ— ${next.height} at ${next.x}, ${next.y}`;
+ void refresh();
});
- root.append(title, meta, label, area, edit, shared);
+ root.append(title, meta, label, area, edit, library, shared);
}
render();