Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

fix(desktop): show the whole capture instead of zooming to the control

Luis Ortiz78b0bdf74e98
diff --git a/AGENTS.md b/AGENTS.md
index 6e97464..75bd6d2 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -581,6 +581,18 @@ control the click landed on is the one that was there before. It is capped at 15
`null`, never an error β€” a slow or unresponsive foreign application costs a step its metadata, not
the recording.
+A desktop step shows its whole frame. `resolveViewport` zooms to `bounds` when a screenshot has
+them, padded by `PAD_RATIO` of the image but never more than `MAX_PAD_MULTIPLE` times the element β€”
+a cap that suits a browser page, where a control is a real fraction of the viewport, and fails on a
+screen grab, where a 30 px window button against 2560 px yields a crop a few hundred pixels wide that
+the card then stretches. So `screenshotForElement` takes `zoomToTarget` and the desktop sink passes
+false, which writes no `bounds` at all; `resolveViewport` falls back to the full image and
+`edits.target` still marks where the click landed. It is written into the data rather than branched
+on `client()`, so a desktop guide frames the same way in whatever opens it, exports included, and the
+extension keeps its zoom because `zoomToTarget` defaults to true. `check:pipeline` asserts the
+viewport equals the frame, and reverting the flag makes it report a 196 Γ— 196 crop of an 800 Γ— 600
+capture β€” the failure in its own numbers.
+
`targetRect` decides what the dashed target in the screenshot encloses. The control's own rectangle
wins when there is one, which is the whole point of reading the accessibility tree; it falls back to
a 28 px box around the click when there is no element, when the rectangle covers more than half the
diff --git a/apps/desktop/src/renderer/capture-sink.ts b/apps/desktop/src/renderer/capture-sink.ts
index 20f5036..bf2dd5d 100644
--- a/apps/desktop/src/renderer/capture-sink.ts
+++ b/apps/desktop/src/renderer/capture-sink.ts
@@ -48,6 +48,7 @@ export class DesktopCaptureSink implements CaptureSink {
height: data.image.height,
cursor: data.cursor ?? null,
targetColor,
+ zoomToTarget: false,
},
meta,
);
diff --git a/apps/desktop/src/renderer/check-pipeline.ts b/apps/desktop/src/renderer/check-pipeline.ts
index 79b153a..6ca794b 100644
--- a/apps/desktop/src/renderer/check-pipeline.ts
+++ b/apps/desktop/src/renderer/check-pipeline.ts
@@ -6,6 +6,7 @@ import { exportGuideAsMarkdown } from '@mimik/core/export/markdown-export';
import { exportGuideAsPDF } from '@mimik/core/export/pdf-export';
import { allScreenshotIds, getGuide, permanentlyDeleteGuide } from '@mimik/core/guides/service';
import { elementSource } from '@mimik/core/guides/types';
+import { resolveViewport } from '@mimik/core/screenshot/geometry';
import { renderScreenshot } from '@mimik/core/screenshot/render';
import { DesktopCaptureSink } from './capture-sink';
@@ -86,6 +87,21 @@ window.mimik.onRequest('mimik:check:verify', async (payload) => {
detail: shot ? `${shot.width} Γ— ${shot.height}, ${shot.blob.size} bytes` : 'no screenshot',
});
+ const viewport = shot ? resolveViewport(shot) : null;
+ results.push({
+ name: 'a desktop step shows its whole frame',
+ ok:
+ shot !== undefined &&
+ shot.bounds === undefined &&
+ viewport !== null &&
+ viewport.width === shot.width &&
+ viewport.height === shot.height &&
+ (shot.edits?.target?.width ?? 0) > 0,
+ detail: shot
+ ? `viewport ${viewport?.width} Γ— ${viewport?.height} of ${shot.width} Γ— ${shot.height}, target still drawn`
+ : 'no screenshot',
+ });
+
results.push({
name: 'description came from the shared heuristic',
ok: (step?.description.length ?? 0) > 0 && step?.descriptionSource === 'heuristic',
diff --git a/packages/core/src/screenshot/__tests__/record.test.ts b/packages/core/src/screenshot/__tests__/record.test.ts
index cbf5ce1..c294975 100644
--- a/packages/core/src/screenshot/__tests__/record.test.ts
+++ b/packages/core/src/screenshot/__tests__/record.test.ts
@@ -1,5 +1,6 @@
import { describe, expect, it } from 'vitest';
import type { ElementMeta } from '@/core/guides/types';
+import { resolveViewport } from '@/core/screenshot/geometry';
import { screenshotForElement } from '@/core/screenshot/record';
const meta: ElementMeta = {
@@ -39,4 +40,11 @@ describe('screenshotForElement', () => {
expect(screenshotForElement(bytes, meta).edits).not.toHaveProperty('cursor');
expect(screenshotForElement({ ...bytes, cursor: null }, meta).edits?.cursor).toBeNull();
});
+
+ it('drops bounds when the target should not be zoomed to, keeping the target itself', () => {
+ const shot = screenshotForElement({ ...bytes, zoomToTarget: false }, meta);
+ expect(shot.bounds).toBeUndefined();
+ expect(shot.edits?.target).toMatchObject({ x: 20, y: 40, width: 200, height: 80 });
+ expect(resolveViewport({ ...shot, blob: new Blob() })).toEqual({ x: 0, y: 0, width: 800, height: 600 });
+ });
});
diff --git a/packages/core/src/screenshot/record.ts b/packages/core/src/screenshot/record.ts
index 3d0af1e..608baf9 100644
--- a/packages/core/src/screenshot/record.ts
+++ b/packages/core/src/screenshot/record.ts
@@ -11,14 +11,17 @@ export interface ScreenshotBytes {
src?: string;
cursor?: CursorMark | null;
targetColor?: string;
+ zoomToTarget?: boolean;
}
export function screenshotForElement(bytes: ScreenshotBytes, meta: ElementMeta): StoredScreenshot {
- const { cursor, targetColor, ...rest } = bytes;
+ const { cursor, targetColor, zoomToTarget = true, ...rest } = bytes;
const ratio = meta.devicePixelRatio;
return {
...rest,
- bounds: { x: meta.rect.x, y: meta.rect.y, width: meta.rect.width, height: meta.rect.height },
+ ...(zoomToTarget
+ ? { bounds: { x: meta.rect.x, y: meta.rect.y, width: meta.rect.width, height: meta.rect.height } }
+ : {}),
pixelRatio: ratio,
clickPoint: meta.clickPoint,
edits: {