Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots 📖

feat(guideme): force manual advance on steps with an edited target or replaced image

Luis Ortizb06cebae2ac3
diff --git a/src/core/guideme/content.ts b/src/core/guideme/content.ts
index ab5286d..ca4ea4b 100644
--- a/src/core/guideme/content.ts
+++ b/src/core/guideme/content.ts
@@ -5,7 +5,7 @@ import { sendMessage } from '@/lib/messaging';
import { findElement } from './finder';
import { GuideMeOverlay } from './overlay';
import type { GuideMeSession } from './session';
-import { BLOCKED_KEY, SESSION_KEY, STEP_KEY } from './session';
+import { BLOCKED_KEY, MANUAL_KEY, SESSION_KEY, STEP_KEY } from './session';
const MAX_RETRIES = 5;
const RETRY_INTERVAL_MS = 1000;
@@ -41,16 +41,16 @@ export class GuideMeController {
}
private async checkForActiveSession() {
- const data = await browser.storage.local.get([SESSION_KEY, STEP_KEY]);
+ const data = await browser.storage.local.get([SESSION_KEY, STEP_KEY, MANUAL_KEY]);
const session = data[SESSION_KEY] as GuideMeSession | null;
const step = data[STEP_KEY] as Step | null;
if (session?.active && step) {
- this.showStep(step, session.activeStepIndex);
+ this.showStep(step, session.activeStepIndex, data[MANUAL_KEY] === true);
}
}
private async onStorageChange() {
- const data = await browser.storage.local.get([SESSION_KEY, STEP_KEY]);
+ const data = await browser.storage.local.get([SESSION_KEY, STEP_KEY, MANUAL_KEY]);
const session = data[SESSION_KEY] as GuideMeSession | null;
const step = data[STEP_KEY] as Step | null;
@@ -62,17 +62,17 @@ export class GuideMeController {
return;
}
- this.showStep(step, session.activeStepIndex);
+ this.showStep(step, session.activeStepIndex, data[MANUAL_KEY] === true);
}
- private showStep(step: Step, stepIndex: number) {
+ private showStep(step: Step, stepIndex: number, requiresManual: boolean) {
this.removeActionDetection();
this.destroyOverlay();
this.stopWatching();
this.currentStepIndex = stepIndex;
const meta = step.elementMeta;
- if (!meta) {
+ if (!meta || requiresManual) {
this.setBlocked(stepIndex);
return;
}
diff --git a/src/core/guideme/session.ts b/src/core/guideme/session.ts
index c9d3c5a..d3a7acb 100644
--- a/src/core/guideme/session.ts
+++ b/src/core/guideme/session.ts
@@ -11,20 +11,32 @@ export interface GuideMeSession {
const SESSION_KEY = 'guideMeSession';
const STEP_KEY = 'guideMeStep';
const BLOCKED_KEY = 'guideMeBlocked';
+const MANUAL_KEY = 'guideMeManual';
-export async function startSession(guideId: string, totalSteps: number, firstStep: Step): Promise<void> {
+export async function startSession(
+ guideId: string,
+ totalSteps: number,
+ firstStep: Step,
+ requiresManual: boolean,
+): Promise<void> {
const session: GuideMeSession = { guideId, activeStepIndex: 0, totalSteps, active: true };
- await localStorage.set({ [SESSION_KEY]: session, [STEP_KEY]: firstStep, [BLOCKED_KEY]: null });
+ await localStorage.set({
+ [SESSION_KEY]: session,
+ [STEP_KEY]: firstStep,
+ [MANUAL_KEY]: requiresManual,
+ [BLOCKED_KEY]: requiresManual ? 0 : null,
+ });
}
-export async function advanceSession(nextStep: Step, nextIndex: number): Promise<void> {
+export async function advanceSession(nextStep: Step, nextIndex: number, requiresManual: boolean): Promise<void> {
const data = await localStorage.get([SESSION_KEY]);
const session = data[SESSION_KEY] as GuideMeSession | undefined;
- if (!session || !session.active) return;
+ if (!session?.active) return;
await localStorage.set({
[SESSION_KEY]: { ...session, activeStepIndex: nextIndex },
[STEP_KEY]: nextStep,
- [BLOCKED_KEY]: null,
+ [MANUAL_KEY]: requiresManual,
+ [BLOCKED_KEY]: requiresManual ? nextIndex : null,
});
}
@@ -35,12 +47,13 @@ export async function completeSession(): Promise<void> {
await localStorage.set({
[SESSION_KEY]: { ...session, active: false },
[STEP_KEY]: null,
+ [MANUAL_KEY]: false,
[BLOCKED_KEY]: null,
});
}
export async function cancelSession(): Promise<void> {
- await localStorage.set({ [SESSION_KEY]: null, [STEP_KEY]: null, [BLOCKED_KEY]: null });
+ await localStorage.set({ [SESSION_KEY]: null, [STEP_KEY]: null, [MANUAL_KEY]: false, [BLOCKED_KEY]: null });
}
export async function getSession(): Promise<GuideMeSession | null> {
@@ -48,4 +61,4 @@ export async function getSession(): Promise<GuideMeSession | null> {
return (data[SESSION_KEY] as GuideMeSession) || null;
}
-export { BLOCKED_KEY, SESSION_KEY, STEP_KEY };
+export { BLOCKED_KEY, MANUAL_KEY, SESSION_KEY, STEP_KEY };
diff --git a/src/core/screenshot/types.ts b/src/core/screenshot/types.ts
index 5dfb947..2072781 100644
--- a/src/core/screenshot/types.ts
+++ b/src/core/screenshot/types.ts
@@ -143,4 +143,5 @@ export interface ScreenshotEdits {
target?: ClickTarget | null;
annotations?: Annotation[];
alt?: string;
+ requiresManual?: boolean;
}
diff --git a/src/entrypoints/background/index.ts b/src/entrypoints/background/index.ts
index c39c53c..f17a8f7 100644
--- a/src/entrypoints/background/index.ts
+++ b/src/entrypoints/background/index.ts
@@ -1,7 +1,14 @@
import { browser, defineBackground, i18n } from '#imports';
import { generateGuideTitle } from '@/core/capture/ai/title';
import { advanceSession, cancelSession, completeSession, getSession, startSession } from '@/core/guideme/session';
-import { createGuide, getGuideDomain, getStepsForGuide, updateGuideTitle } from '@/core/guides/service';
+import {
+ createGuide,
+ getGuideDomain,
+ getScreenshotsForSteps,
+ getStepsForGuide,
+ updateGuideTitle,
+} from '@/core/guides/service';
+import type { Step } from '@/core/guides/types';
import { getActiveTab, localStorage, sendMessageToTab, setSidePanelBehavior, updateTab } from '@/lib/browser-api';
import { logger } from '@/lib/logger';
import { onMessage } from '@/lib/messaging';
@@ -51,6 +58,13 @@ async function generateTitleInBackground(guideId: string) {
}
}
+async function stepRequiresManual(step: Step): Promise<boolean> {
+ if (!step.elementMeta) return true;
+ if (!step.screenshotId) return false;
+ const screenshots = await getScreenshotsForSteps([step.screenshotId]);
+ return screenshots.get(step.id)?.edits?.requiresManual === true;
+}
+
export default defineBackground(() => {
logger.info('Background service worker started');
@@ -184,7 +198,7 @@ export default defineBackground(() => {
const firstStep = steps.find((s) => s.elementMeta) ?? steps[0];
if (!steps.some((s) => s.elementMeta)) return { started: false, error: 'Guide lacks element metadata' };
- await startSession(data.guideId, steps.length, firstStep);
+ await startSession(data.guideId, steps.length, firstStep, await stepRequiresManual(firstStep));
const activeTab = await getActiveTab();
if (activeTab?.id && firstStep.url) {
@@ -212,7 +226,7 @@ export default defineBackground(() => {
await completeSession();
return { advanced: true, completed: true };
}
- await advanceSession(nextStep, nextIndex);
+ await advanceSession(nextStep, nextIndex, await stepRequiresManual(nextStep));
const currentTab = await getActiveTab();
if (currentTab?.id && nextStep.url && nextStep.url !== currentTab.url) {
@@ -238,7 +252,7 @@ export default defineBackground(() => {
const prevIndex = data.stepIndex - 1;
const prevStep = steps[prevIndex];
if (!prevStep) return { moved: false };
- await advanceSession(prevStep, prevIndex);
+ await advanceSession(prevStep, prevIndex, await stepRequiresManual(prevStep));
const currentTab = await getActiveTab();
if (currentTab?.id && prevStep.url && prevStep.url !== currentTab.url) {
diff --git a/src/locales/en.yml b/src/locales/en.yml
index 013e813..3afe7e4 100644
--- a/src/locales/en.yml
+++ b/src/locales/en.yml
@@ -45,6 +45,7 @@ library:
guideme:
live: Live
roadblock: "We hit a roadblock. Try taking action on the screen to move forward."
+ manualStep: "This step was edited, so it cannot be detected automatically. Mark it complete when you are done."
markComplete: "Mark as complete"
stepOf: "Step $1 of $2"
noDescription: No description
diff --git a/src/locales/es.yml b/src/locales/es.yml
index 8d65f29..cd7e86c 100644
--- a/src/locales/es.yml
+++ b/src/locales/es.yml
@@ -45,6 +45,7 @@ library:
guideme:
live: En vivo
roadblock: "Nos encontramos con un obstáculo. Actúa en la pantalla para avanzar."
+ manualStep: "Este paso fue editado, así que no se puede detectar automáticamente. Márcalo como completado cuando termines."
markComplete: "Marcar como completado"
stepOf: "Paso $1 de $2"
noDescription: Sin descripción
diff --git a/src/locales/fr.yml b/src/locales/fr.yml
index 7078b40..8fdcf36 100644
--- a/src/locales/fr.yml
+++ b/src/locales/fr.yml
@@ -45,6 +45,7 @@ library:
guideme:
live: En direct
roadblock: "Nous avons rencontré un obstacle. Agissez à l'écran pour continuer."
+ manualStep: "Cette étape a été modifiée, elle ne peut donc pas être détectée automatiquement. Marquez-la comme terminée une fois faite."
markComplete: "Marquer comme terminé"
stepOf: "Étape $1 sur $2"
noDescription: Aucune description
diff --git a/src/locales/pt-BR.yml b/src/locales/pt-BR.yml
index 1623ea2..76bb764 100644
--- a/src/locales/pt-BR.yml
+++ b/src/locales/pt-BR.yml
@@ -45,6 +45,7 @@ library:
guideme:
live: Ao vivo
roadblock: "Encontramos um obstáculo. Aja na tela para avançar."
+ manualStep: "Esta etapa foi editada, então não pode ser detectada automaticamente. Marque como concluída quando terminar."
markComplete: "Marcar como concluído"
stepOf: "Passo $1 de $2"
noDescription: Sem descrição
diff --git a/src/ui/shared/AnnotationEditor.tsx b/src/ui/shared/AnnotationEditor.tsx
index a5c827e..3953ba5 100644
--- a/src/ui/shared/AnnotationEditor.tsx
+++ b/src/ui/shared/AnnotationEditor.tsx
@@ -36,6 +36,7 @@ import {
import type {
Annotation,
ArrowEnd,
+ ClickTarget,
FontFamily,
LineWidth,
ScreenshotEdits,
@@ -285,6 +286,18 @@ function handleCorners(b: ScreenshotBounds): [Handle, number, number][] {
];
}
+function sameTarget(a: ClickTarget | null, b: ClickTarget | null): boolean {
+ if (!a || !b) return a === b;
+ return (
+ Math.round(a.x) === Math.round(b.x) &&
+ Math.round(a.y) === Math.round(b.y) &&
+ Math.round(a.width) === Math.round(b.width) &&
+ Math.round(a.height) === Math.round(b.height) &&
+ a.color === b.color &&
+ a.border === b.border
+ );
+}
+
function selectionBounds(a: Annotation, scale: number): ScreenshotBounds {
const inset = SELECTION_GAP * scale;
const b = annotationBounds(a);
@@ -396,6 +409,7 @@ export default function AnnotationEditor({ screenshot, tool, onDone, onCancel }:
setAnnotations((prev) => prev.map((a) => (a.id === TARGET_ID && a.type === 'target' ? { ...a, ...patch } : a)));
};
+ const initialTargetRef = useRef<ClickTarget | null>(resolveTarget(screenshot));
const canvasRef = useRef<HTMLCanvasElement>(null);
const textInputRef = useRef<HTMLInputElement>(null);
const dragRef = useRef<DragState | null>(null);
@@ -862,20 +876,22 @@ export default function AnnotationEditor({ screenshot, tool, onDone, onCancel }:
const handleDone = async () => {
setSaving(true);
const targetAnnotation = annotations.find((a) => a.id === TARGET_ID);
+ const nextTarget =
+ targetAnnotation && targetAnnotation.type === 'target'
+ ? {
+ x: targetAnnotation.x,
+ y: targetAnnotation.y,
+ width: targetAnnotation.w,
+ height: targetAnnotation.h,
+ border: targetAnnotation.border,
+ color: targetAnnotation.color,
+ }
+ : null;
const nextEdits: ScreenshotEdits = {
...screenshot.edits,
annotations: annotations.filter((a) => a.id !== TARGET_ID),
- target:
- targetAnnotation && targetAnnotation.type === 'target'
- ? {
- x: targetAnnotation.x,
- y: targetAnnotation.y,
- width: targetAnnotation.w,
- height: targetAnnotation.h,
- border: targetAnnotation.border,
- color: targetAnnotation.color,
- }
- : null,
+ target: nextTarget,
+ requiresManual: screenshot.edits?.requiresManual || !sameTarget(initialTargetRef.current, nextTarget),
};
if (viewport) nextEdits.viewport = viewport;
await updateScreenshotEdits(screenshot.id, nextEdits);
diff --git a/src/ui/shared/ScreenshotView.tsx b/src/ui/shared/ScreenshotView.tsx
index 1c24c22..a280db4 100644
--- a/src/ui/shared/ScreenshotView.tsx
+++ b/src/ui/shared/ScreenshotView.tsx
@@ -235,7 +235,7 @@ export default function ScreenshotView({
const { width, height } = bitmap;
bitmap.close();
- const nextEdits: ScreenshotEdits = { ...effectiveEdits, target: null };
+ const nextEdits: ScreenshotEdits = { ...effectiveEdits, target: null, requiresManual: true };
delete nextEdits.viewport;
const nextScreenshot: Screenshot = { ...baseScreenshot, blob: file, mimeType: file.type, width, height };
diff --git a/src/ui/sidepanel/GuideMeView.tsx b/src/ui/sidepanel/GuideMeView.tsx
index 98804c7..9b17692 100644
--- a/src/ui/sidepanel/GuideMeView.tsx
+++ b/src/ui/sidepanel/GuideMeView.tsx
@@ -2,7 +2,7 @@ import { ArrowLeft, Check, ChevronLeft, ChevronRight, TriangleAlert } from 'luci
import { useCallback, useEffect, useState } from 'react';
import { browser, i18n } from '#imports';
import type { GuideMeSession } from '@/core/guideme/session';
-import { BLOCKED_KEY, SESSION_KEY } from '@/core/guideme/session';
+import { BLOCKED_KEY, MANUAL_KEY, SESSION_KEY } from '@/core/guideme/session';
import { getGuide } from '@/core/guides/service';
import type { Guide, Screenshot, Step } from '@/core/guides/types';
import { sendMessage } from '@/lib/messaging';
@@ -70,6 +70,7 @@ export default function GuideMeView({ guideId, onExit, onComplete }: GuideMeView
const [viewedStepIndex, setViewedStepIndex] = useState(0);
const [showExitConfirm, setShowExitConfirm] = useState(false);
const [blockedStepIndex, setBlockedStepIndex] = useState<number | null>(null);
+ const [isManualStep, setIsManualStep] = useState(false);
const loadGuide = useCallback(async () => {
const result = await getGuide(guideId);
@@ -90,6 +91,9 @@ export default function GuideMeView({ guideId, onExit, onComplete }: GuideMeView
if (changes[BLOCKED_KEY]) {
setBlockedStepIndex((changes[BLOCKED_KEY].newValue as number | null) ?? null);
}
+ if (changes[MANUAL_KEY]) {
+ setIsManualStep(changes[MANUAL_KEY].newValue === true);
+ }
if (!changes[SESSION_KEY]) return;
const session = changes[SESSION_KEY].newValue as GuideMeSession | null;
if (!session) return;
@@ -106,13 +110,14 @@ export default function GuideMeView({ guideId, onExit, onComplete }: GuideMeView
}, [guideId, onComplete]);
useEffect(() => {
- browser.storage.local.get([SESSION_KEY, BLOCKED_KEY]).then((result: Record<string, unknown>) => {
+ browser.storage.local.get([SESSION_KEY, BLOCKED_KEY, MANUAL_KEY]).then((result: Record<string, unknown>) => {
const session = result[SESSION_KEY] as GuideMeSession | null;
if (session?.active) {
setActiveStepIndex(session.activeStepIndex);
setViewedStepIndex(session.activeStepIndex);
}
setBlockedStepIndex((result[BLOCKED_KEY] as number | null) ?? null);
+ setIsManualStep(result[MANUAL_KEY] === true);
});
}, []);
@@ -189,7 +194,7 @@ export default function GuideMeView({ guideId, onExit, onComplete }: GuideMeView
<div className="mx-4 mb-3 rounded-lg bg-secondary p-3">
<p className="flex items-start gap-2 text-[12px] leading-relaxed text-foreground">
<TriangleAlert size={14} className="shrink-0 mt-0.5 text-accent" />
- {i18n.t('guideme.roadblock')}
+ {isManualStep ? i18n.t('guideme.manualStep') : i18n.t('guideme.roadblock')}
</p>
<button
onClick={() => sendMessage('guideMeStepCompleted', { stepIndex: activeStepIndex }).catch(() => {})}