π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
fix(voice): start narration mid-recording and keep ai out of narrated steps
Luis Ortizee6c365d59cc
diff --git a/src/core/guides/__tests__/ai-pending.test.ts b/src/core/guides/__tests__/ai-pending.test.tsindex 8462ce8..aa14ed8 100644--- a/src/core/guides/__tests__/ai-pending.test.ts+++ b/src/core/guides/__tests__/ai-pending.test.ts@@ -24,7 +24,13 @@ const { broadcasts } = vi.hoisted(() => {});import { db } from '../db';-import { applyNarrationToSteps, clearStepAiPending, type GuideChangeEvent, updateStepDescription } from '../service';+import {+ applyAiDescription,+ applyNarrationToSteps,+ clearStepAiPending,+ type GuideChangeEvent,+ updateStepDescription,+} from '../service';import type { Guide, Step } from '../types';const FALLBACK = 'Clicked Save';@@ -193,3 +199,27 @@ describe('clearStepAiPending', () => {expect(sibling?.aiPending).toBe(true);});});++describe('applyAiDescription', () => {+ it('fills a step that narration did not cover', async () => {+ await db.steps.add(makeStep({ id: 's9', aiPending: false }));++ await applyAiDescription('s9', AI_TEXT);++ const step = await db.steps.get('s9');+ expect(step?.description).toBe(AI_TEXT);+ expect(step?.descriptionSource).toBe('ai');+ });++ it('leaves a narrated step alone', async () => {+ await db.steps.add(makeStep({ id: 's10', description: 'What I said out loud', descriptionSource: 'narration' }));++ await applyAiDescription('s10', AI_TEXT);++ expect((await db.steps.get('s10'))?.description).toBe('What I said out loud');+ });++ it('does nothing for a step that no longer exists', async () => {+ await expect(applyAiDescription('gone', AI_TEXT)).resolves.toBeUndefined();+ });+});diff --git a/src/core/guides/service.ts b/src/core/guides/service.tsindex b7c88a5..725b966 100644--- a/src/core/guides/service.ts+++ b/src/core/guides/service.ts@@ -209,6 +209,16 @@ export async function applyNarrationToSteps(updates: readonly NarrationUpdate[])notifyGuidesChanged({ type: 'mutated' });}+export async function applyAiDescription(stepId: string, description: string): Promise<void> {+ const wrote = await db.transaction('rw', db.steps, async () => {+ const step = await db.steps.get(stepId);+ if (!step || step.descriptionSource === 'narration') return false;+ await db.steps.update(stepId, { description, descriptionSource: 'ai', aiPending: false });+ return true;+ });+ if (wrote) notifyGuidesChanged({ type: 'mutated' });+}+export async function clearStepAiPending(stepId: string, description?: string): Promise<void> {const wrote = await db.transaction('rw', db.steps, async () => {const step = await db.steps.get(stepId);diff --git a/src/entrypoints/background/__tests__/deferred-descriptions.test.ts b/src/entrypoints/background/__tests__/deferred-descriptions.test.tsnew file mode 100644index 0000000..793e5bd--- /dev/null+++ b/src/entrypoints/background/__tests__/deferred-descriptions.test.ts@@ -0,0 +1,82 @@+import { beforeEach, describe, expect, it } from 'vitest';+import type { DOMContext } from '@/core/capture/dom/context';+import {+ clearDeferredDescriptions,+ deferDescription,+ shouldQueueAiDescription,+ takeDeferredDescriptions,+} from '../deferred-descriptions';++const ctx = (name: string) => ({ page: { title: name, path: '/' } }) as DOMContext;++const base = { action: 'click', hasDomContext: true, hasAiKey: true, narrationCapturing: false };++describe('shouldQueueAiDescription', () => {+ it('queues a description for a normal click', () => {+ expect(shouldQueueAiDescription(base)).toBe(true);+ });++ it('does not queue while narration is capturing', () => {+ expect(shouldQueueAiDescription({ ...base, narrationCapturing: true })).toBe(false);+ });++ it('does not queue without an api key', () => {+ expect(shouldQueueAiDescription({ ...base, hasAiKey: false })).toBe(false);+ });++ it('does not queue without dom context', () => {+ expect(shouldQueueAiDescription({ ...base, hasDomContext: false })).toBe(false);+ });++ it('does not queue for input actions', () => {+ expect(shouldQueueAiDescription({ ...base, action: 'input' })).toBe(false);+ });+});++describe('takeDeferredDescriptions', () => {+ beforeEach(() => {+ clearDeferredDescriptions('g1');+ clearDeferredDescriptions('g2');+ });++ it('returns the steps narration did not cover', () => {+ deferDescription('g1', 's1', ctx('one'));+ deferDescription('g1', 's2', ctx('two'));++ const pending = takeDeferredDescriptions('g1', ['s1']);++ expect(pending.map((p) => p.stepId)).toEqual(['s2']);+ });++ it('returns every step when nothing was narrated', () => {+ deferDescription('g1', 's1', ctx('one'));+ deferDescription('g1', 's2', ctx('two'));++ expect(takeDeferredDescriptions('g1', []).map((p) => p.stepId)).toEqual(['s1', 's2']);+ });++ it('carries the dom context captured at the time', () => {+ deferDescription('g1', 's1', ctx('billing'));++ expect(takeDeferredDescriptions('g1', [])[0].domContext.page.title).toBe('billing');+ });++ it('drains, so a second call returns nothing', () => {+ deferDescription('g1', 's1', ctx('one'));+ takeDeferredDescriptions('g1', []);++ expect(takeDeferredDescriptions('g1', [])).toEqual([]);+ });++ it('keeps guides separate', () => {+ deferDescription('g1', 's1', ctx('one'));+ deferDescription('g2', 's2', ctx('two'));++ expect(takeDeferredDescriptions('g1', []).map((p) => p.stepId)).toEqual(['s1']);+ expect(takeDeferredDescriptions('g2', []).map((p) => p.stepId)).toEqual(['s2']);+ });++ it('returns nothing for a guide that deferred nothing', () => {+ expect(takeDeferredDescriptions('g1', [])).toEqual([]);+ });+});diff --git a/src/entrypoints/background/__tests__/start-narration.test.ts b/src/entrypoints/background/__tests__/start-narration.test.tsnew file mode 100644index 0000000..bc630bb--- /dev/null+++ b/src/entrypoints/background/__tests__/start-narration.test.ts@@ -0,0 +1,25 @@+import { describe, expect, it } from 'vitest';+import { CaptureState } from '@/core/capture/machine';+import { canStartNarrationNow } from '../voice';++describe('canStartNarrationNow', () => {+ it('starts narration when a recording is running and narration is not', () => {+ expect(canStartNarrationNow(CaptureState.RECORDING, 'idle')).toBe(true);+ });++ it('does not start when nothing is being recorded', () => {+ expect(canStartNarrationNow(CaptureState.IDLE, 'idle')).toBe(false);+ });++ it('does not start a second capture when narration is already running', () => {+ expect(canStartNarrationNow(CaptureState.RECORDING, 'recording')).toBe(false);+ });++ it('does not start while a previous take is still transcribing', () => {+ expect(canStartNarrationNow(CaptureState.RECORDING, 'transcribing')).toBe(false);+ });++ it('starts again after a failed take', () => {+ expect(canStartNarrationNow(CaptureState.RECORDING, 'error')).toBe(true);+ });+});diff --git a/src/entrypoints/background/ai-description.ts b/src/entrypoints/background/ai-description.tsnew file mode 100644index 0000000..85fca81--- /dev/null+++ b/src/entrypoints/background/ai-description.ts@@ -0,0 +1,13 @@+import { getAIDescription } from '@/core/capture/ai/description';+import type { DOMContext } from '@/core/capture/dom/context';+import { localStorage } from '@/lib/browser-api';++export async function generateAiDescription(domContext: DOMContext): Promise<string | undefined> {+ const settings = await localStorage.get(['aiApiKey', 'aiProvider', 'aiModel']);+ if (!settings.aiApiKey) return undefined;++ const provider = (settings.aiProvider as string) || 'openai';+ const model = (settings.aiModel as string) || 'gpt-4o-mini';+ const description = await getAIDescription(domContext, provider, model, settings.aiApiKey as string);+ return description || undefined;+}diff --git a/src/entrypoints/background/deferred-descriptions.ts b/src/entrypoints/background/deferred-descriptions.tsnew file mode 100644index 0000000..af98748--- /dev/null+++ b/src/entrypoints/background/deferred-descriptions.ts@@ -0,0 +1,45 @@+import type { DOMContext } from '@/core/capture/dom/context';++export interface DeferredDescription {+ stepId: string;+ domContext: DOMContext;+}++export interface AiDescriptionInput {+ action: string;+ hasDomContext: boolean;+ hasAiKey: boolean;+ narrationCapturing: boolean;+}++const deferred = new Map<string, Map<string, DOMContext>>();++export function shouldQueueAiDescription({+ action,+ hasDomContext,+ hasAiKey,+ narrationCapturing,+}: AiDescriptionInput): boolean {+ return action !== 'input' && hasDomContext && hasAiKey && !narrationCapturing;+}++export function deferDescription(guideId: string, stepId: string, domContext: DOMContext): void {+ const forGuide = deferred.get(guideId) ?? new Map<string, DOMContext>();+ forGuide.set(stepId, domContext);+ deferred.set(guideId, forGuide);+}++export function takeDeferredDescriptions(guideId: string, narratedStepIds: readonly string[]): DeferredDescription[] {+ const forGuide = deferred.get(guideId);+ if (!forGuide) return [];+ deferred.delete(guideId);++ const narrated = new Set(narratedStepIds);+ return [...forGuide]+ .filter(([stepId]) => !narrated.has(stepId))+ .map(([stepId, domContext]) => ({ stepId, domContext }));+}++export function clearDeferredDescriptions(guideId: string): void {+ deferred.delete(guideId);+}diff --git a/src/entrypoints/background/describe-unnarrated.ts b/src/entrypoints/background/describe-unnarrated.tsnew file mode 100644index 0000000..adc5e28--- /dev/null+++ b/src/entrypoints/background/describe-unnarrated.ts@@ -0,0 +1,13 @@+import { applyAiDescription } from '@/core/guides/service';+import { generateAiDescription } from './ai-description';+import { takeDeferredDescriptions } from './deferred-descriptions';+import { queueDescription } from './description-queue';++export function describeUnnarratedSteps(guideId: string, narratedStepIds: readonly string[]): void {+ for (const { stepId, domContext } of takeDeferredDescriptions(guideId, narratedStepIds)) {+ queueDescription(guideId, async () => {+ const description = await generateAiDescription(domContext);+ if (description) await applyAiDescription(stepId, description);+ });+ }+}diff --git a/src/entrypoints/background/index.ts b/src/entrypoints/background/index.tsindex 4932840..e4d5e1c 100644--- a/src/entrypoints/background/index.ts+++ b/src/entrypoints/background/index.ts@@ -28,7 +28,13 @@ import { generateDescriptionOnDemand, generateGuideMetaOnStop, settlePendingDescimport { registerNavigationListeners } from './navigation';import { handleCaptureStep, handleFinalizeInputStep, handleUpdateInputStep } from './step-pipeline';import { broadcastStartCapture, broadcastStopCapture, showNotificationOnTab } from './tab-manager';-import { getVoiceUpdate, registerVoiceListeners, startVoiceNarration, stopVoiceNarration } from './voice';+import {+ canStartNarrationNow,+ getVoiceUpdate,+ registerVoiceListeners,+ startVoiceNarration,+ stopVoiceNarration,+} from './voice';async function resolveManual(step: Step): Promise<boolean> {if (!step.screenshotId) return stepRequiresManual(step, null);@@ -142,6 +148,17 @@ export default defineBackground(() => {return { success: true, guideId: guideId ?? undefined, inserted: false };});+ onMessage('startNarration', async () => {+ await waitUntilReady();+ const actor = getActor();+ if (!canStartNarrationNow(String(actor.getSnapshot().value), getVoiceUpdate().phase)) {+ return { started: false };+ }+ const activeTab = await getActiveTab();+ await startVoiceNarration(activeTab?.id);+ return { started: getVoiceUpdate().phase === 'recording' };+ });+onMessage('enterBlurMode', async () => {await waitUntilReady();await broadcastStopCapture();diff --git a/src/entrypoints/background/step-pipeline.ts b/src/entrypoints/background/step-pipeline.tsindex c9a37a1..cc1c381 100644--- a/src/entrypoints/background/step-pipeline.ts+++ b/src/entrypoints/background/step-pipeline.ts@@ -1,4 +1,3 @@-import { getAIDescription } from '@/core/capture/ai/description';import type { DOMContext } from '@/core/capture/dom/context';import { CaptureState } from '@/core/capture/machine';import { buildFallbackDescription } from '@/core/capture/step-description';@@ -16,7 +15,10 @@ import { captureVisibleTab, localStorage } from '@/lib/browser-api';import { logger } from '@/lib/logger';import type { CaptureStepData, CaptureStepResponse } from '@/lib/messaging';import { getActor } from './actor';+import { generateAiDescription } from './ai-description';+import { deferDescription, shouldQueueAiDescription } from './deferred-descriptions';import { queueDescription } from './description-queue';+import { getVoiceUpdate } from './voice';async function takeScreenshot(stepId: string, meta: ElementMeta): Promise<string | undefined> {try {@@ -55,14 +57,9 @@ async function takeScreenshot(stepId: string, meta: ElementMeta): Promise<string}async function tryAIDescription(stepId: string, domContext: DOMContext) {- const settings = await localStorage.get(['aiApiKey', 'aiProvider', 'aiModel']);- if (!settings.aiApiKey) return;-- const provider = (settings.aiProvider as string) || 'openai';- const model = (settings.aiModel as string) || 'gpt-4o-mini';+ if (!(await localStorage.get(['aiApiKey'])).aiApiKey) return;try {- const description = await getAIDescription(domContext, provider, model, settings.aiApiKey as string);- await clearStepAiPending(stepId, description || undefined);+ await clearStepAiPending(stepId, await generateAiDescription(domContext));} catch (err) {await clearStepAiPending(stepId);throw err;@@ -81,7 +78,14 @@ export async function handleCaptureStep(data: CaptureStepData): Promise<CaptureSconst screenshotId = await takeScreenshot(stepId, data.elementMeta);- const willUseAI = data.action !== 'input' && !!data.domContext && !!(await localStorage.get(['aiApiKey'])).aiApiKey;+ const narrationCapturing = getVoiceUpdate().phase === 'recording';+ const hasAiKey = !!(await localStorage.get(['aiApiKey'])).aiApiKey;+ const willUseAI = shouldQueueAiDescription({+ action: data.action,+ hasDomContext: !!data.domContext,+ hasAiKey,+ narrationCapturing,+ });await createStep({id: stepId,@@ -99,7 +103,8 @@ export async function handleCaptureStep(data: CaptureStepData): Promise<CaptureSconst domContext = data.domContext;if (data.action !== 'input' && domContext) {- queueDescription(guideId, () => tryAIDescription(stepId, domContext));+ if (willUseAI) queueDescription(guideId, () => tryAIDescription(stepId, domContext));+ else if (narrationCapturing && hasAiKey) deferDescription(guideId, stepId, domContext);}return { stepId };diff --git a/src/entrypoints/background/voice.ts b/src/entrypoints/background/voice.tsindex f705994..89bd79c 100644--- a/src/entrypoints/background/voice.ts+++ b/src/entrypoints/background/voice.ts@@ -1,3 +1,4 @@+import { CaptureState } from '@/core/capture/machine';import { hasVoiceApiKey, VOICE_KEY_SETTINGS } from '@/core/capture/voice/api-key';import { narrationUpdates } from '@/core/capture/voice/narration-updates';import { applyNarrationToSteps, findExistingStepIds, getStepsForGuide } from '@/core/guides/service';@@ -15,6 +16,7 @@ import {stopVoiceCapture,supportsVoice,} from '@/lib/offscreen';+import type { VoicePhase } from '@/lib/port';import { broadcastVoiceToPanel, type PanelVoiceUpdate } from '@/lib/port';import {isVoiceMessageFor,@@ -29,6 +31,7 @@ import {} from '@/lib/voice-messages';import { narrateRecording, readTranscriptionSettings, type VoiceRecording } from '@/lib/voice-narration';import { handedOffPcm, voiceStopAction } from '@/lib/voice-recovery';+import { describeUnnarratedSteps } from './describe-unnarrated';const START_TIMEOUT_MS = 8000;@@ -102,6 +105,10 @@ async function beginVoiceCapture(microphoneId: string | undefined, tabId: numberawait closeVoiceHost();}+export function canStartNarrationNow(captureState: string, voicePhase: VoicePhase): boolean {+ return captureState === CaptureState.RECORDING && voicePhase !== 'recording' && voicePhase !== 'transcribing';+}+export async function startVoiceNarration(tabId?: number): Promise<void> {if (!supportsVoice()) {logger.warn('voice: no microphone host available in this browser, recording without narration');@@ -188,7 +195,7 @@ export async function stopVoiceNarration(guideId: string): Promise<void> {}}-async function applyNarration(_guideId: string, result: VoiceResultEvent['result']): Promise<void> {+async function applyNarration(guideId: string, result: VoiceResultEvent['result']): Promise<void> {try {transcribingGuideId = null;const narrated = result.descriptions.map((entry) => entry.stepId);@@ -197,6 +204,10 @@ async function applyNarration(_guideId: string, result: VoiceResultEvent['resultawait applyNarrationToSteps(updates);logger.info('voice: narration applied', { narrated: updates.length, of: narrated.length, stats: result.stats });report({ phase: 'idle', narrated: updates.length });+ describeUnnarratedSteps(+ guideId,+ updates.map((update) => update.stepId),+ );} catch (error) {logger.error('voice: narration could not be applied', error);report({ phase: 'error', reason: 'unknown', error: String(error) });diff --git a/src/lib/messaging.ts b/src/lib/messaging.tsindex 694c5ec..6600c5b 100644--- a/src/lib/messaging.ts+++ b/src/lib/messaging.ts@@ -121,6 +121,10 @@ export interface EnterBlurModeResponse {entered: boolean;}+export interface StartNarrationResponse {+ started: boolean;+}+export interface ExitBlurModeResponse {exited: boolean;}@@ -138,6 +142,7 @@ interface MimikProtocol {guideMeGoTo(data: GuideMe_GoToData): GuideMe_GoToResponse;enterBlurMode(): EnterBlurModeResponse;exitBlurMode(): ExitBlurModeResponse;+ startNarration(): StartNarrationResponse;generateGuideDescription(data: GenerateGuideDescriptionData): GenerateGuideDescriptionResponse;validateApiKey(data: ValidateApiKeyData): ValidateApiKeyResponse;rewriteSelection(data: RewriteSelectionData): RewriteSelectionResponse;diff --git a/src/ui/sidepanel/MicToggle.tsx b/src/ui/sidepanel/MicToggle.tsxindex 31cb8d1..50de1c9 100644--- a/src/ui/sidepanel/MicToggle.tsx+++ b/src/ui/sidepanel/MicToggle.tsx@@ -3,6 +3,7 @@ import { useCallback, useEffect, useState } from 'react';import { browser, i18n } from '#imports';import { hasVoiceApiKey, VOICE_KEY_SETTINGS } from '@/core/capture/voice/api-key';import { getActiveTab, localStorage } from '@/lib/browser-api';+import { sendMessage } from '@/lib/messaging';import { abortVoiceCapture, openMicPermissionPage } from '@/lib/offscreen';import { Tooltip, TooltipContent, TooltipTrigger } from '@/ui/components/ui/tooltip';@@ -59,7 +60,10 @@ export default function MicToggle({ enabled, live, onChange }: MicToggleProps) {return;}- if (await microphoneGranted()) return;+ if (await microphoneGranted()) {+ await sendMessage('startNarration', undefined).catch(() => undefined);+ return;+ }const tab = await getActiveTab();await openMicPermissionPage(tab?.id).catch(() => undefined);}, [enabled, live, locked, onChange]);