🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots 📖
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
feat(guideme): force manual advance on steps with an edited target or replaced image
Luis Ortizb06cebae2ac3
diff --git a/src/core/guideme/content.ts b/src/core/guideme/content.tsindex ab5286d..ca4ea4b 100644--- a/src/core/guideme/content.ts+++ b/src/core/guideme/content.ts@@ -5,7 +5,7 @@ import { sendMessage } from '@/lib/messaging';import { findElement } from './finder';import { GuideMeOverlay } from './overlay';import type { GuideMeSession } from './session';-import { BLOCKED_KEY, SESSION_KEY, STEP_KEY } from './session';+import { BLOCKED_KEY, MANUAL_KEY, SESSION_KEY, STEP_KEY } from './session';const MAX_RETRIES = 5;const RETRY_INTERVAL_MS = 1000;@@ -41,16 +41,16 @@ export class GuideMeController {}private async checkForActiveSession() {- const data = await browser.storage.local.get([SESSION_KEY, STEP_KEY]);+ const data = await browser.storage.local.get([SESSION_KEY, STEP_KEY, MANUAL_KEY]);const session = data[SESSION_KEY] as GuideMeSession | null;const step = data[STEP_KEY] as Step | null;if (session?.active && step) {- this.showStep(step, session.activeStepIndex);+ this.showStep(step, session.activeStepIndex, data[MANUAL_KEY] === true);}}private async onStorageChange() {- const data = await browser.storage.local.get([SESSION_KEY, STEP_KEY]);+ const data = await browser.storage.local.get([SESSION_KEY, STEP_KEY, MANUAL_KEY]);const session = data[SESSION_KEY] as GuideMeSession | null;const step = data[STEP_KEY] as Step | null;@@ -62,17 +62,17 @@ export class GuideMeController {return;}- this.showStep(step, session.activeStepIndex);+ this.showStep(step, session.activeStepIndex, data[MANUAL_KEY] === true);}- private showStep(step: Step, stepIndex: number) {+ private showStep(step: Step, stepIndex: number, requiresManual: boolean) {this.removeActionDetection();this.destroyOverlay();this.stopWatching();this.currentStepIndex = stepIndex;const meta = step.elementMeta;- if (!meta) {+ if (!meta || requiresManual) {this.setBlocked(stepIndex);return;}diff --git a/src/core/guideme/session.ts b/src/core/guideme/session.tsindex c9d3c5a..d3a7acb 100644--- a/src/core/guideme/session.ts+++ b/src/core/guideme/session.ts@@ -11,20 +11,32 @@ export interface GuideMeSession {const SESSION_KEY = 'guideMeSession';const STEP_KEY = 'guideMeStep';const BLOCKED_KEY = 'guideMeBlocked';+const MANUAL_KEY = 'guideMeManual';-export async function startSession(guideId: string, totalSteps: number, firstStep: Step): Promise<void> {+export async function startSession(+ guideId: string,+ totalSteps: number,+ firstStep: Step,+ requiresManual: boolean,+): Promise<void> {const session: GuideMeSession = { guideId, activeStepIndex: 0, totalSteps, active: true };- await localStorage.set({ [SESSION_KEY]: session, [STEP_KEY]: firstStep, [BLOCKED_KEY]: null });+ await localStorage.set({+ [SESSION_KEY]: session,+ [STEP_KEY]: firstStep,+ [MANUAL_KEY]: requiresManual,+ [BLOCKED_KEY]: requiresManual ? 0 : null,+ });}-export async function advanceSession(nextStep: Step, nextIndex: number): Promise<void> {+export async function advanceSession(nextStep: Step, nextIndex: number, requiresManual: boolean): Promise<void> {const data = await localStorage.get([SESSION_KEY]);const session = data[SESSION_KEY] as GuideMeSession | undefined;- if (!session || !session.active) return;+ if (!session?.active) return;await localStorage.set({[SESSION_KEY]: { ...session, activeStepIndex: nextIndex },[STEP_KEY]: nextStep,- [BLOCKED_KEY]: null,+ [MANUAL_KEY]: requiresManual,+ [BLOCKED_KEY]: requiresManual ? nextIndex : null,});}@@ -35,12 +47,13 @@ export async function completeSession(): Promise<void> {await localStorage.set({[SESSION_KEY]: { ...session, active: false },[STEP_KEY]: null,+ [MANUAL_KEY]: false,[BLOCKED_KEY]: null,});}export async function cancelSession(): Promise<void> {- await localStorage.set({ [SESSION_KEY]: null, [STEP_KEY]: null, [BLOCKED_KEY]: null });+ await localStorage.set({ [SESSION_KEY]: null, [STEP_KEY]: null, [MANUAL_KEY]: false, [BLOCKED_KEY]: null });}export async function getSession(): Promise<GuideMeSession | null> {@@ -48,4 +61,4 @@ export async function getSession(): Promise<GuideMeSession | null> {return (data[SESSION_KEY] as GuideMeSession) || null;}-export { BLOCKED_KEY, SESSION_KEY, STEP_KEY };+export { BLOCKED_KEY, MANUAL_KEY, SESSION_KEY, STEP_KEY };diff --git a/src/core/screenshot/types.ts b/src/core/screenshot/types.tsindex 5dfb947..2072781 100644--- a/src/core/screenshot/types.ts+++ b/src/core/screenshot/types.ts@@ -143,4 +143,5 @@ export interface ScreenshotEdits {target?: ClickTarget | null;annotations?: Annotation[];alt?: string;+ requiresManual?: boolean;}diff --git a/src/entrypoints/background/index.ts b/src/entrypoints/background/index.tsindex c39c53c..f17a8f7 100644--- a/src/entrypoints/background/index.ts+++ b/src/entrypoints/background/index.ts@@ -1,7 +1,14 @@import { browser, defineBackground, i18n } from '#imports';import { generateGuideTitle } from '@/core/capture/ai/title';import { advanceSession, cancelSession, completeSession, getSession, startSession } from '@/core/guideme/session';-import { createGuide, getGuideDomain, getStepsForGuide, updateGuideTitle } from '@/core/guides/service';+import {+ createGuide,+ getGuideDomain,+ getScreenshotsForSteps,+ getStepsForGuide,+ updateGuideTitle,+} from '@/core/guides/service';+import type { Step } from '@/core/guides/types';import { getActiveTab, localStorage, sendMessageToTab, setSidePanelBehavior, updateTab } from '@/lib/browser-api';import { logger } from '@/lib/logger';import { onMessage } from '@/lib/messaging';@@ -51,6 +58,13 @@ async function generateTitleInBackground(guideId: string) {}}+async function stepRequiresManual(step: Step): Promise<boolean> {+ if (!step.elementMeta) return true;+ if (!step.screenshotId) return false;+ const screenshots = await getScreenshotsForSteps([step.screenshotId]);+ return screenshots.get(step.id)?.edits?.requiresManual === true;+}+export default defineBackground(() => {logger.info('Background service worker started');@@ -184,7 +198,7 @@ export default defineBackground(() => {const firstStep = steps.find((s) => s.elementMeta) ?? steps[0];if (!steps.some((s) => s.elementMeta)) return { started: false, error: 'Guide lacks element metadata' };- await startSession(data.guideId, steps.length, firstStep);+ await startSession(data.guideId, steps.length, firstStep, await stepRequiresManual(firstStep));const activeTab = await getActiveTab();if (activeTab?.id && firstStep.url) {@@ -212,7 +226,7 @@ export default defineBackground(() => {await completeSession();return { advanced: true, completed: true };}- await advanceSession(nextStep, nextIndex);+ await advanceSession(nextStep, nextIndex, await stepRequiresManual(nextStep));const currentTab = await getActiveTab();if (currentTab?.id && nextStep.url && nextStep.url !== currentTab.url) {@@ -238,7 +252,7 @@ export default defineBackground(() => {const prevIndex = data.stepIndex - 1;const prevStep = steps[prevIndex];if (!prevStep) return { moved: false };- await advanceSession(prevStep, prevIndex);+ await advanceSession(prevStep, prevIndex, await stepRequiresManual(prevStep));const currentTab = await getActiveTab();if (currentTab?.id && prevStep.url && prevStep.url !== currentTab.url) {diff --git a/src/locales/en.yml b/src/locales/en.ymlindex 013e813..3afe7e4 100644--- a/src/locales/en.yml+++ b/src/locales/en.yml@@ -45,6 +45,7 @@ library:guideme:live: Liveroadblock: "We hit a roadblock. Try taking action on the screen to move forward."+ manualStep: "This step was edited, so it cannot be detected automatically. Mark it complete when you are done."markComplete: "Mark as complete"stepOf: "Step $1 of $2"noDescription: No descriptiondiff --git a/src/locales/es.yml b/src/locales/es.ymlindex 8d65f29..cd7e86c 100644--- a/src/locales/es.yml+++ b/src/locales/es.yml@@ -45,6 +45,7 @@ library:guideme:live: En vivoroadblock: "Nos encontramos con un obstáculo. Actúa en la pantalla para avanzar."+ manualStep: "Este paso fue editado, así que no se puede detectar automáticamente. Márcalo como completado cuando termines."markComplete: "Marcar como completado"stepOf: "Paso $1 de $2"noDescription: Sin descripcióndiff --git a/src/locales/fr.yml b/src/locales/fr.ymlindex 7078b40..8fdcf36 100644--- a/src/locales/fr.yml+++ b/src/locales/fr.yml@@ -45,6 +45,7 @@ library:guideme:live: En directroadblock: "Nous avons rencontré un obstacle. Agissez à l'écran pour continuer."+ manualStep: "Cette étape a été modifiée, elle ne peut donc pas être détectée automatiquement. Marquez-la comme terminée une fois faite."markComplete: "Marquer comme terminé"stepOf: "Étape $1 sur $2"noDescription: Aucune descriptiondiff --git a/src/locales/pt-BR.yml b/src/locales/pt-BR.ymlindex 1623ea2..76bb764 100644--- a/src/locales/pt-BR.yml+++ b/src/locales/pt-BR.yml@@ -45,6 +45,7 @@ library:guideme:live: Ao vivoroadblock: "Encontramos um obstáculo. Aja na tela para avançar."+ manualStep: "Esta etapa foi editada, então não pode ser detectada automaticamente. Marque como concluída quando terminar."markComplete: "Marcar como concluído"stepOf: "Passo $1 de $2"noDescription: Sem descriçãodiff --git a/src/ui/shared/AnnotationEditor.tsx b/src/ui/shared/AnnotationEditor.tsxindex a5c827e..3953ba5 100644--- a/src/ui/shared/AnnotationEditor.tsx+++ b/src/ui/shared/AnnotationEditor.tsx@@ -36,6 +36,7 @@ import {import type {Annotation,ArrowEnd,+ ClickTarget,FontFamily,LineWidth,ScreenshotEdits,@@ -285,6 +286,18 @@ function handleCorners(b: ScreenshotBounds): [Handle, number, number][] {];}+function sameTarget(a: ClickTarget | null, b: ClickTarget | null): boolean {+ if (!a || !b) return a === b;+ return (+ Math.round(a.x) === Math.round(b.x) &&+ Math.round(a.y) === Math.round(b.y) &&+ Math.round(a.width) === Math.round(b.width) &&+ Math.round(a.height) === Math.round(b.height) &&+ a.color === b.color &&+ a.border === b.border+ );+}+function selectionBounds(a: Annotation, scale: number): ScreenshotBounds {const inset = SELECTION_GAP * scale;const b = annotationBounds(a);@@ -396,6 +409,7 @@ export default function AnnotationEditor({ screenshot, tool, onDone, onCancel }:setAnnotations((prev) => prev.map((a) => (a.id === TARGET_ID && a.type === 'target' ? { ...a, ...patch } : a)));};+ const initialTargetRef = useRef<ClickTarget | null>(resolveTarget(screenshot));const canvasRef = useRef<HTMLCanvasElement>(null);const textInputRef = useRef<HTMLInputElement>(null);const dragRef = useRef<DragState | null>(null);@@ -862,20 +876,22 @@ export default function AnnotationEditor({ screenshot, tool, onDone, onCancel }:const handleDone = async () => {setSaving(true);const targetAnnotation = annotations.find((a) => a.id === TARGET_ID);+ const nextTarget =+ targetAnnotation && targetAnnotation.type === 'target'+ ? {+ x: targetAnnotation.x,+ y: targetAnnotation.y,+ width: targetAnnotation.w,+ height: targetAnnotation.h,+ border: targetAnnotation.border,+ color: targetAnnotation.color,+ }+ : null;const nextEdits: ScreenshotEdits = {...screenshot.edits,annotations: annotations.filter((a) => a.id !== TARGET_ID),- target:- targetAnnotation && targetAnnotation.type === 'target'- ? {- x: targetAnnotation.x,- y: targetAnnotation.y,- width: targetAnnotation.w,- height: targetAnnotation.h,- border: targetAnnotation.border,- color: targetAnnotation.color,- }- : null,+ target: nextTarget,+ requiresManual: screenshot.edits?.requiresManual || !sameTarget(initialTargetRef.current, nextTarget),};if (viewport) nextEdits.viewport = viewport;await updateScreenshotEdits(screenshot.id, nextEdits);diff --git a/src/ui/shared/ScreenshotView.tsx b/src/ui/shared/ScreenshotView.tsxindex 1c24c22..a280db4 100644--- a/src/ui/shared/ScreenshotView.tsx+++ b/src/ui/shared/ScreenshotView.tsx@@ -235,7 +235,7 @@ export default function ScreenshotView({const { width, height } = bitmap;bitmap.close();- const nextEdits: ScreenshotEdits = { ...effectiveEdits, target: null };+ const nextEdits: ScreenshotEdits = { ...effectiveEdits, target: null, requiresManual: true };delete nextEdits.viewport;const nextScreenshot: Screenshot = { ...baseScreenshot, blob: file, mimeType: file.type, width, height };diff --git a/src/ui/sidepanel/GuideMeView.tsx b/src/ui/sidepanel/GuideMeView.tsxindex 98804c7..9b17692 100644--- a/src/ui/sidepanel/GuideMeView.tsx+++ b/src/ui/sidepanel/GuideMeView.tsx@@ -2,7 +2,7 @@ import { ArrowLeft, Check, ChevronLeft, ChevronRight, TriangleAlert } from 'luciimport { useCallback, useEffect, useState } from 'react';import { browser, i18n } from '#imports';import type { GuideMeSession } from '@/core/guideme/session';-import { BLOCKED_KEY, SESSION_KEY } from '@/core/guideme/session';+import { BLOCKED_KEY, MANUAL_KEY, SESSION_KEY } from '@/core/guideme/session';import { getGuide } from '@/core/guides/service';import type { Guide, Screenshot, Step } from '@/core/guides/types';import { sendMessage } from '@/lib/messaging';@@ -70,6 +70,7 @@ export default function GuideMeView({ guideId, onExit, onComplete }: GuideMeViewconst [viewedStepIndex, setViewedStepIndex] = useState(0);const [showExitConfirm, setShowExitConfirm] = useState(false);const [blockedStepIndex, setBlockedStepIndex] = useState<number | null>(null);+ const [isManualStep, setIsManualStep] = useState(false);const loadGuide = useCallback(async () => {const result = await getGuide(guideId);@@ -90,6 +91,9 @@ export default function GuideMeView({ guideId, onExit, onComplete }: GuideMeViewif (changes[BLOCKED_KEY]) {setBlockedStepIndex((changes[BLOCKED_KEY].newValue as number | null) ?? null);}+ if (changes[MANUAL_KEY]) {+ setIsManualStep(changes[MANUAL_KEY].newValue === true);+ }if (!changes[SESSION_KEY]) return;const session = changes[SESSION_KEY].newValue as GuideMeSession | null;if (!session) return;@@ -106,13 +110,14 @@ export default function GuideMeView({ guideId, onExit, onComplete }: GuideMeView}, [guideId, onComplete]);useEffect(() => {- browser.storage.local.get([SESSION_KEY, BLOCKED_KEY]).then((result: Record<string, unknown>) => {+ browser.storage.local.get([SESSION_KEY, BLOCKED_KEY, MANUAL_KEY]).then((result: Record<string, unknown>) => {const session = result[SESSION_KEY] as GuideMeSession | null;if (session?.active) {setActiveStepIndex(session.activeStepIndex);setViewedStepIndex(session.activeStepIndex);}setBlockedStepIndex((result[BLOCKED_KEY] as number | null) ?? null);+ setIsManualStep(result[MANUAL_KEY] === true);});}, []);@@ -189,7 +194,7 @@ export default function GuideMeView({ guideId, onExit, onComplete }: GuideMeView<div className="mx-4 mb-3 rounded-lg bg-secondary p-3"><p className="flex items-start gap-2 text-[12px] leading-relaxed text-foreground"><TriangleAlert size={14} className="shrink-0 mt-0.5 text-accent" />- {i18n.t('guideme.roadblock')}+ {isManualStep ? i18n.t('guideme.manualStep') : i18n.t('guideme.roadblock')}</p><buttononClick={() => sendMessage('guideMeStepCompleted', { stepIndex: activeStepIndex }).catch(() => {})}