Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

feat: advanced capture engine with event merging, click interception, and drag detection

Luis Ortizecfd7e5b7a60
diff --git a/src/background/step-description.ts b/src/background/step-description.ts
index 51bb59b..7864ccc 100644
--- a/src/background/step-description.ts
+++ b/src/background/step-description.ts
@@ -16,6 +16,7 @@ export function buildFallbackDescription(action: string, meta: ElementMeta): str
switch (action) {
case 'click':
+ case 'auxclick':
if (meta.tag === 'input' && meta.inputType === 'checkbox') return `Toggle checkbox ${target}`;
if (meta.tag === 'input' && meta.inputType === 'radio') return `Select radio ${target}`;
if (meta.href) return `Click link "${target}"`;
@@ -23,8 +24,14 @@ export function buildFallbackDescription(action: string, meta: ElementMeta): str
case 'input':
if (meta.inputType) return `Type into ${meta.inputType} field ${target}`;
return `Type into ${target}`;
- case 'scroll':
- return 'Scroll the page';
+ case 'copy':
+ return `Copy from ${target}`;
+ case 'paste':
+ return `Paste into ${target}`;
+ case 'cut':
+ return `Cut from ${target}`;
+ case 'drag':
+ return `Drag ${target}`;
case 'navigate':
return 'Navigate to page';
default:
diff --git a/src/content/events.ts b/src/content/events.ts
index f3d8177..c7ad51f 100644
--- a/src/content/events.ts
+++ b/src/content/events.ts
@@ -1,6 +1,12 @@
import { extractElementMeta } from './element-meta';
-const FOCUSABLE_SELECTOR = 'a[href], button, input, select, textarea, [role="button"], [role="link"], [role="tab"], [role="menuitem"], [role="checkbox"], [role="radio"], [tabindex]';
+const FOCUSABLE_SELECTOR = 'a[href], button, input, select, textarea, [role="button"], [role="link"], [role="tab"], [role="menuitem"], [role="checkbox"], [role="radio"], [tabindex], [contenteditable="true"]';
+
+const MERGE_MAX_ABSORB = 6;
+const MERGE_WINDOW_MS = 500;
+const MERGE_WINDOW_MS = 5;
+
+const DRAG_MIN_PX = 40;
const DEDUP_MS = 200;
let lastClickTarget: Element | null = null;
@@ -9,15 +15,133 @@ let lastClickTime = 0;
function findFocusableAncestor(el: Element): HTMLElement {
const focusable = el.closest(FOCUSABLE_SELECTOR);
if (focusable instanceof HTMLElement) return focusable;
-
if (el instanceof HTMLElement) return el;
if (el.parentElement instanceof HTMLElement) return el.parentElement;
-
return document.body;
}
+function isTextField(el: Element): boolean {
+ if (el instanceof HTMLInputElement) {
+ const textTypes = ['text', 'email', 'password', 'search', 'tel', 'url', 'number'];
+ return textTypes.includes(el.type);
+ }
+ return el instanceof HTMLTextAreaElement
+ || (el instanceof HTMLElement && el.isContentEditable);
+}
+
+function isNavigatingClick(el: HTMLElement): boolean {
+ const anchor = el.closest('a[href]');
+ if (!anchor) return false;
+ const href = anchor.getAttribute('href');
+ if (!href || href === '#' || href.startsWith('javascript:')) return false;
+ return true;
+}
+
+interface MergeWindow {
+ target: Element;
+ bounds: DOMRect;
+}
+
+function timeGap(a: DOMRect, b: DOMRect): number {
+ const dx = Math.max(a.left - (b.left + b.width), b.left - (a.left + a.width), 0);
+ const dy = Math.max(a.top - (b.top + b.height), b.top - (a.top + a.height), 0);
+ return Math.sqrt(dx * dx + dy * dy);
+}
+
+function sameTarget(a: DOMRect, b: DOMRect): boolean {
+ return Math.abs(a.left - b.left) <= MERGE_WINDOW_MS
+ && Math.abs(a.top - b.top) <= MERGE_WINDOW_MS;
+}
+
+function shouldAbsorb(entries: MergeWindow[], target: Element, bounds: DOMRect): boolean {
+ if (entries.length >= MERGE_MAX_ABSORB) return false;
+ if (entries.some(e => e.target === target)) return false;
+ if (entries.length > 0 && !sameTarget(entries[0].bounds, bounds)) return false;
+ if (entries.some(e => timeGap(e.bounds, bounds) > MERGE_WINDOW_MS)) return false;
+ return true;
+}
+
+const MAX_ELEMENT_RATIO = 0.8;
+
+function createHighlightOverlay(): HTMLDivElement {
+ const el = document.createElement('div');
+ el.setAttribute('data-mimik-ignore', '');
+ el.style.cssText = `
+ position: fixed;
+ pointer-events: none;
+ border: 3px solid #3B82F6;
+ border-radius: 6px;
+ z-index: 2147483647;
+ box-sizing: border-box;
+ opacity: 0;
+ transition: opacity 0.15s ease;
+ `;
+ document.documentElement.appendChild(el);
+ return el;
+}
+
+function isTooLarge(el: Element): boolean {
+ const rect = el.getBoundingClientRect();
+ return (rect.width / window.innerWidth > MAX_ELEMENT_RATIO)
+ && (rect.height / window.innerHeight > MAX_ELEMENT_RATIO);
+}
+
export function startCapture(guideId: string): () => void {
- let scrollTimer: ReturnType<typeof setTimeout> | null = null;
+ let highlightEl: HTMLDivElement | null = null;
+ let hoveredElement: Element | null = null;
+ let hideTimer: ReturnType<typeof setTimeout> | null = null;
+ let rafId: number | null = null;
+
+ let mergeWin: MergeWindow[] = [];
+ let mergeTimer: ReturnType<typeof setTimeout> | null = null;
+
+ let dragStartX: number | null = null;
+ let dragStartY: number | null = null;
+ let dragStartElement: Element | null = null;
+
+ let eventQueue: Promise<void> = Promise.resolve();
+
+ function enqueue(fn: () => void) {
+ eventQueue = eventQueue.then(() => {
+ try { fn(); } catch (err) { console.warn('[Mimik] Event handler error', err); }
+ });
+ }
+
+ function showHighlight(target: Element) {
+ if (!highlightEl) highlightEl = createHighlightOverlay();
+ if (hideTimer) { clearTimeout(hideTimer); hideTimer = null; }
+ const rect = target.getBoundingClientRect();
+ highlightEl.style.top = `${rect.top - 3}px`;
+ highlightEl.style.left = `${rect.left - 3}px`;
+ highlightEl.style.width = `${rect.width + 6}px`;
+ highlightEl.style.height = `${rect.height + 6}px`;
+ highlightEl.style.opacity = '1';
+ }
+
+ function hideHighlight() {
+ if (highlightEl) highlightEl.style.opacity = '0';
+ }
+
+ function handleMouseOver(e: MouseEvent) {
+ const rawTarget = e.target;
+ if (!rawTarget || !(rawTarget instanceof Element)) return;
+ if (rawTarget.closest('[data-mimik-ignore]')) return;
+ const target = findFocusableAncestor(rawTarget);
+ if (target === document.body || target === document.documentElement) return;
+ if (isTooLarge(target)) return;
+ if (target === hoveredElement) return;
+ hoveredElement = target;
+ if (rafId) cancelAnimationFrame(rafId);
+ rafId = requestAnimationFrame(() => showHighlight(target));
+ }
+
+ function handleMouseOut(e: MouseEvent) {
+ const related = e.relatedTarget;
+ if (related instanceof Element && hoveredElement?.contains(related)) return;
+ hoveredElement = null;
+ if (hideTimer) clearTimeout(hideTimer);
+ hideTimer = setTimeout(hideHighlight, 80);
+ }
function sendAction(action: string, meta: ReturnType<typeof extractElementMeta>) {
try {
@@ -32,6 +156,33 @@ export function startCapture(guideId: string): () => void {
}
}
+ function resetWindow() {
+ if (mergeTimer) { clearTimeout(mergeTimer); mergeTimer = null; }
+ mergeWin = [];
+ }
+
+ function tryMergeOrSend(target: HTMLElement, action: string) {
+ const bounds = target.getBoundingClientRect();
+
+ if (mergeWin.length > 0 && shouldAbsorb(mergeWin, target, bounds)) {
+ mergeWin.push({ target, bounds });
+ if (mergeTimer) clearTimeout(mergeTimer);
+ mergeTimer = setTimeout(resetWindow, 600);
+ return;
+ }
+
+ resetWindow();
+ mergeWin.push({ target, bounds });
+ mergeTimer = setTimeout(resetWindow, 600);
+
+ try {
+ const meta = extractElementMeta(target);
+ sendAction(action, meta);
+ } catch (err) {
+ console.warn('[Mimik] Failed to capture action', err);
+ }
+ }
+
function handleClick(e: MouseEvent) {
const rawTarget = e.target;
if (!rawTarget || !(rawTarget instanceof Element)) return;
@@ -44,61 +195,151 @@ export function startCapture(guideId: string): () => void {
lastClickTarget = target;
lastClickTime = now;
- try {
- const meta = extractElementMeta(target);
- sendAction('click', meta);
- } catch (err) {
- console.warn('[Mimik] Failed to capture click', err);
+ if (!isTextField(target) && !(target instanceof HTMLSelectElement) && !(target instanceof HTMLOptionElement)) {
+ if (isNavigatingClick(target)) {
+ e.preventDefault();
+ e.stopImmediatePropagation();
+
+ enqueue(() => tryMergeOrSend(target, 'click'));
+
+ const anchor = target.closest('a[href]') as HTMLAnchorElement;
+ if (anchor) {
+ setTimeout(() => {
+ window.location.href = anchor.href;
+ }, 150);
+ }
+ return;
+ }
}
+
+ enqueue(() => tryMergeOrSend(target, 'click'));
+ }
+
+ function handleAuxClick(e: MouseEvent) {
+ const rawTarget = e.target;
+ if (!rawTarget || !(rawTarget instanceof Element)) return;
+ const target = findFocusableAncestor(rawTarget);
+ if (target.closest('[data-mimik-ignore]')) return;
+
+ enqueue(() => tryMergeOrSend(target, 'auxclick'));
}
function handleKeydown(e: KeyboardEvent) {
- if (!['Enter', 'Escape', 'Tab'].includes(e.key)) return;
const target = e.target instanceof HTMLElement ? e.target : document.activeElement;
if (!target || !(target instanceof HTMLElement)) return;
+ if (target.closest('[data-mimik-ignore]')) return;
- try {
- const meta = extractElementMeta(target);
- sendAction(`keydown:${e.key}`, meta);
- } catch (err) {
- console.warn('[Mimik] Failed to capture keydown', err);
- }
+ enqueue(() => {
+ try {
+ const meta = extractElementMeta(target);
+ sendAction(`keydown:${e.key}`, meta);
+ } catch (err) {
+ console.warn('[Mimik] Failed to capture keydown', err);
+ }
+ });
}
function handleInput(e: Event) {
- if (!(e.target instanceof HTMLInputElement || e.target instanceof HTMLTextAreaElement || e.target instanceof HTMLSelectElement)) return;
- try {
- const meta = extractElementMeta(e.target as HTMLElement);
- sendAction('input', meta);
- } catch (err) {
- console.warn('[Mimik] Failed to capture input', err);
- }
+ const target = e.target;
+ if (!target || !(target instanceof HTMLElement)) return;
+ if (!(target instanceof HTMLInputElement
+ || target instanceof HTMLTextAreaElement
+ || target instanceof HTMLSelectElement
+ || target.isContentEditable)) return;
+
+ enqueue(() => {
+ try {
+ const meta = extractElementMeta(target);
+ sendAction('input', meta);
+ } catch (err) {
+ console.warn('[Mimik] Failed to capture input', err);
+ }
+ });
}
- function handleScroll() {
- if (scrollTimer) clearTimeout(scrollTimer);
- scrollTimer = setTimeout(() => {
- sendAction('scroll', {
- tag: 'window', cssSelector: 'window', textContent: null,
- ariaLabel: null, placeholder: null, altText: null,
- name: null, role: 'scrollbar', href: null,
- inputType: null, dataTestId: null,
- rect: { x: 0, y: 0, width: window.innerWidth, height: window.innerHeight },
- devicePixelRatio: window.devicePixelRatio,
- });
- }, 500);
+ function handleClipboard(e: ClipboardEvent) {
+ const target = e.target instanceof HTMLElement ? e.target : document.activeElement;
+ if (!target || !(target instanceof HTMLElement)) return;
+ if (target.closest('[data-mimik-ignore]')) return;
+
+ enqueue(() => {
+ try {
+ const meta = extractElementMeta(target);
+ sendAction(e.type, meta); // 'copy', 'paste', or 'cut'
+ } catch (err) {
+ console.warn('[Mimik] Failed to capture clipboard', err);
+ }
+ });
+ }
+
+ function handleMouseDown(e: MouseEvent) {
+ dragStartX = e.pageX;
+ dragStartY = e.pageY;
+ dragStartElement = e.target instanceof Element ? e.target : null;
+ }
+
+ function handleMouseUp(e: MouseEvent) {
+ if (dragStartX == null || dragStartY == null || !dragStartElement) {
+ dragStartX = dragStartY = null;
+ dragStartElement = null;
+ return;
+ }
+
+ const dx = Math.abs(e.pageX - dragStartX);
+ const dy = Math.abs(e.pageY - dragStartY);
+
+ if (dx >= DRAG_MIN_PX || dy >= DRAG_MIN_PX) {
+ const target = findFocusableAncestor(dragStartElement);
+ if (!target.closest('[data-mimik-ignore]')) {
+ enqueue(() => {
+ try {
+ const meta = extractElementMeta(target);
+ sendAction('drag', meta);
+ } catch (err) {
+ console.warn('[Mimik] Failed to capture drag', err);
+ }
+ });
+ }
+ }
+
+ dragStartX = dragStartY = null;
+ dragStartElement = null;
}
window.addEventListener('click', handleClick, { capture: true });
+ window.addEventListener('auxclick', handleAuxClick, { capture: true });
window.addEventListener('keydown', handleKeydown, { capture: true });
window.addEventListener('input', handleInput, { capture: true });
- window.addEventListener('scroll', handleScroll, { capture: true, passive: true });
+ window.addEventListener('copy', handleClipboard as EventListener, { capture: true });
+ window.addEventListener('paste', handleClipboard as EventListener, { capture: true });
+ window.addEventListener('cut', handleClipboard as EventListener, { capture: true });
+
+ window.addEventListener('mousedown', handleMouseDown, { capture: true });
+ window.addEventListener('mouseup', handleMouseUp, { capture: true });
+
+ window.addEventListener('mouseover', handleMouseOver, { capture: true });
+ window.addEventListener('mouseout', handleMouseOut, { capture: true });
return () => {
window.removeEventListener('click', handleClick, { capture: true });
+ window.removeEventListener('auxclick', handleAuxClick, { capture: true });
window.removeEventListener('keydown', handleKeydown, { capture: true });
window.removeEventListener('input', handleInput, { capture: true });
- window.removeEventListener('scroll', handleScroll, { capture: true });
- if (scrollTimer) clearTimeout(scrollTimer);
+ window.removeEventListener('copy', handleClipboard as EventListener, { capture: true });
+ window.removeEventListener('paste', handleClipboard as EventListener, { capture: true });
+ window.removeEventListener('cut', handleClipboard as EventListener, { capture: true });
+ window.removeEventListener('mousedown', handleMouseDown, { capture: true });
+ window.removeEventListener('mouseup', handleMouseUp, { capture: true });
+ window.removeEventListener('mouseover', handleMouseOver, { capture: true });
+ window.removeEventListener('mouseout', handleMouseOut, { capture: true });
+ if (hideTimer) clearTimeout(hideTimer);
+ if (rafId) cancelAnimationFrame(rafId);
+ if (mergeTimer) clearTimeout(mergeTimer);
+ if (highlightEl) {
+ highlightEl.remove();
+ highlightEl = null;
+ }
+ hoveredElement = null;
+ mergeWin = [];
};
}