π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
feat(desktop): say on the recording card why an AI description failed
Luis Ortiza82b0dc4ee4d
diff --git a/apps/desktop/scripts/check-overlay.ts b/apps/desktop/scripts/check-overlay.tsindex da5c552..dddaad2 100644--- a/apps/desktop/scripts/check-overlay.ts+++ b/apps/desktop/scripts/check-overlay.ts@@ -274,6 +274,19 @@ app.whenReady().then(async () => {shortTitle.height === veil.height && longTitle.height === veil.height && longTitle.preview < shortTitle.preview,`${veil.height} px while capturing, ${shortTitle.height} px with a short title, ${longTitle.height} px with a long one, preview ${shortTitle.preview} β ${longTitle.preview} px`,);+ overlay.setAiFailure({ reason: 'rejected', provider: 'openai' });+ await settle();+ const failed = await card(+ "shown: !document.querySelector('#aiNotice').hidden, text: document.querySelector('#aiNotice').textContent",+ );+ overlay.setAiFailure(null);+ await settle();+ const cleared = await card("shown: !document.querySelector('#aiNotice').hidden");+ check(+ 'a failed AI description says why, in the extension words',+ failed.shown === true && String(failed.text).includes('OpenAI') && cleared.shown === false,+ `notice: ${failed.text}; after clearing shown: ${cleared.shown}`,+ );const whenIdle = await windowWithHash('controls')?.webContents.executeJavaScript("document.querySelector('#primary').disabled",);diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.tsindex 3e45736..dabbb07 100644--- a/apps/desktop/src/main/index.ts+++ b/apps/desktop/src/main/index.ts@@ -6,7 +6,7 @@ import { focusedWindow } from './capture/focused-window';import { DesktopRecorder, frameFor } from './capture/recorder';import { registerScreenshotProtocol, SCREENSHOT_SCHEME, sweepScreenshots } from './capture/screenshot-store';import { type CaptureMode, type CaptureSettings, loadSettings, saveSettings } from './capture/settings';-import { CaptureOverlay, type OverlayCommand, type OverlayStep } from './overlay';+import { CaptureOverlay, type OverlayAiFailure, type OverlayCommand, type OverlayStep } from './overlay';import { bindShortcuts, type ShortcutName, shortcutMap, unbindShortcuts } from './shortcuts';import { checkForUpdates } from './updater';@@ -248,6 +248,7 @@ async function onOverlayCommand(command: OverlayCommand): Promise<void> {}if (command === 'start' && !guideId) {steps = [];+ overlay?.setAiFailure(null);try {guideId = await ask<string>(mainWindow?.webContents ?? null, 'mimik:capture:startGuide');} catch (error) {@@ -374,16 +375,20 @@ if (!app.requestSingleInstanceLock()) {applyShortcuts();return captureSettings;});- ipcMain.on('mimik:capture:described', (_event, stepId: string, description: string | null) => {- const step = steps.find((candidate) => candidate.id === stepId);- if (!step) return;- if (description) {- step.title = description;- step.source = 'ai';- }- step.pending = false;- if (step === steps.at(-1)) overlay?.showStep(step);- });+ ipcMain.on(+ 'mimik:capture:described',+ (_event, stepId: string, description: string | null, failure: OverlayAiFailure | null) => {+ if (failure) overlay?.setAiFailure(failure);+ const step = steps.find((candidate) => candidate.id === stepId);+ if (!step) return;+ if (description) {+ step.title = description;+ step.source = 'ai';+ }+ step.pending = false;+ if (step === steps.at(-1)) overlay?.showStep(step);+ },+ );ipcMain.handle('mimik:capture:region', () => overlay?.region);ipcMain.handle('mimik:capture:edit', () => {enterCapture();diff --git a/apps/desktop/src/main/overlay/index.ts b/apps/desktop/src/main/overlay/index.tsindex ad07ce0..5b63a96 100644--- a/apps/desktop/src/main/overlay/index.ts+++ b/apps/desktop/src/main/overlay/index.ts@@ -1,4 +1,5 @@import { join } from 'node:path';+import type { AiFailureReason } from '@mimik/core/capture/ai/errors';import { BrowserWindow, ipcMain, screen } from 'electron';import { clampToDisplays, loadRegion, type Region, saveRegion } from '../capture/region';import type { CaptureMode } from '../capture/settings';@@ -33,6 +34,11 @@ export interface OverlayShortcuts {capture: string | null;}+export interface OverlayAiFailure {+ reason: AiFailureReason;+ provider: string;+}+export interface OverlayView {state: OverlayState;region: Region;@@ -41,6 +47,7 @@ export interface OverlayView {busy: boolean;starting: boolean;shortcuts: OverlayShortcuts;+ aiFailure: OverlayAiFailure | null;}export interface OverlayOptions {@@ -112,6 +119,7 @@ export class CaptureOverlay {private step: OverlayStep | null = null;private size = { width: CONTROLS.width, height: CONTROLS.height };private busy = false;+ private aiFailure: OverlayAiFailure | null = null;private editingFrom: OverlayState = 'hidden';private modeBeforeEdit: CaptureMode | null = null;private hiding: { shown: BrowserWindow[]; ready: Promise<unknown>; users: number } | null = null;@@ -157,6 +165,7 @@ export class CaptureOverlay {busy: this.busy,starting: this.starting,shortcuts: this.options.shortcuts?.() ?? NO_SHORTCUTS,+ aiFailure: this.aiFailure,};}@@ -183,6 +192,11 @@ export class CaptureOverlay {this.broadcast();}+ setAiFailure(failure: OverlayAiFailure | null): void {+ this.aiFailure = failure;+ this.broadcast();+ }+setBusy(busy: boolean): void {if (this.busy === busy) return;this.busy = busy;diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.tsindex 5506a5b..3a3c6c6 100644--- a/apps/desktop/src/preload/index.ts+++ b/apps/desktop/src/preload/index.ts@@ -1,5 +1,6 @@import { contextBridge, ipcRenderer } from 'electron';import type { CaptureSettings } from '../main/capture/settings';+import type { OverlayAiFailure } from '../main/overlay';interface Region {x: number;@@ -14,8 +15,8 @@ const api = {region: (): Promise<Region> => ipcRenderer.invoke('mimik:capture:region'),edit: (): Promise<void> => ipcRenderer.invoke('mimik:capture:edit'),arm: (): Promise<void> => ipcRenderer.invoke('mimik:capture:arm'),- described: (stepId: string, description: string | null): void =>- ipcRenderer.send('mimik:capture:described', stepId, description),+ described: (stepId: string, description: string | null, failure: OverlayAiFailure | null): void =>+ ipcRenderer.send('mimik:capture:described', stepId, description, failure),settings: {get: (): Promise<CaptureSettings> => ipcRenderer.invoke('mimik:capture:settings:get'),set: (patch: Partial<CaptureSettings>): Promise<CaptureSettings> =>diff --git a/apps/desktop/src/renderer/ai/describe-step.ts b/apps/desktop/src/renderer/ai/describe-step.tsindex 8e448dc..8d424b3 100644--- a/apps/desktop/src/renderer/ai/describe-step.ts+++ b/apps/desktop/src/renderer/ai/describe-step.ts@@ -1,20 +1,29 @@import { getAIDescription } from '@mimik/core/capture/ai/description';+import { describeAiFailure } from '@mimik/core/capture/ai/errors';import { serializeScreenContext } from '@mimik/core/capture/ai/screen-context';import type { ElementMeta } from '@mimik/core/guides/types';+import { logger } from '@mimik/core/logger';+import type { OverlayAiFailure } from '../../main/overlay';import { credentials } from './credentials';-export async function describeStep(action: string, meta: ElementMeta): Promise<string | null> {+export async function describeStep(+ action: string,+ meta: ElementMeta,+): Promise<{ text: string | null; failure: OverlayAiFailure | null }> {const keys = await credentials();- if (!keys) return null;+ if (!keys) return { text: null, failure: null };try {- return await getAIDescription(+ const text = await getAIDescription(serializeScreenContext(action, meta),keys.provider,keys.model,keys.apiKey,keys.baseUrl,);- } catch {- return null;+ return { text: text || null, failure: null };+ } catch (err) {+ const failure = describeAiFailure(err);+ logger.error(`AI description failed (${failure.reason}${failure.status ? ` ${failure.status}` : ''})`, err);+ return { text: null, failure: { reason: failure.reason, provider: keys.provider } };}}diff --git a/apps/desktop/src/renderer/capture-sink.ts b/apps/desktop/src/renderer/capture-sink.tsindex 801c1c6..ef76658 100644--- a/apps/desktop/src/renderer/capture-sink.ts+++ b/apps/desktop/src/renderer/capture-sink.ts@@ -86,10 +86,10 @@ export class DesktopCaptureSink implements CaptureSink {await addStepToGuide(data.guideId, stepId);if (pending) {- void describeStep(data.action, meta).then(async (written) => {- if (written) await updateStepDescription(stepId, written, 'ai');+ void describeStep(data.action, meta).then(async ({ text, failure }) => {+ if (text) await updateStepDescription(stepId, text, 'ai');await clearStepAiPending(stepId);- window.mimik.capture.described(stepId, written);+ window.mimik.capture.described(stepId, text, failure);});}diff --git a/apps/desktop/src/renderer/icons.ts b/apps/desktop/src/renderer/icons.tsindex 71f5e25..cdad930 100644--- a/apps/desktop/src/renderer/icons.ts+++ b/apps/desktop/src/renderer/icons.ts@@ -11,6 +11,7 @@ import {Pause,Play,Trash2,+ TriangleAlert,Video,} from 'lucide';@@ -27,6 +28,7 @@ const ICONS = {monitor: Monitor,area: Crop,keyboard: Keyboard,+ alert: TriangleAlert,};export type IconName = keyof typeof ICONS;diff --git a/apps/desktop/src/renderer/overlay.css b/apps/desktop/src/renderer/overlay.cssindex 7d83d52..c9d9c2b 100644--- a/apps/desktop/src/renderer/overlay.css+++ b/apps/desktop/src/renderer/overlay.css@@ -484,6 +484,31 @@ body.controls #skeleton .meta span {}}+body.controls #aiNotice {+ display: flex;+ align-items: flex-start;+ gap: 7px;+ padding: 10px 14px 0;+ font-size: 11px;+ line-height: 1.45;+ color: var(--muted);+}++body.controls #aiNotice svg {+ flex-shrink: 0;+ margin-top: 1px;+ color: #dc2626;+}++body.controls #aiNotice p {+ margin: 0;+}++body.controls #aiNotice strong {+ font-weight: 600;+ color: var(--deep);+}+body.controls #stepMeta,body.controls #skeleton .meta {display: flex;diff --git a/apps/desktop/src/renderer/overlay/controls.ts b/apps/desktop/src/renderer/overlay/controls.tsindex 5c99982..e2efe26 100644--- a/apps/desktop/src/renderer/overlay/controls.ts+++ b/apps/desktop/src/renderer/overlay/controls.ts@@ -1,3 +1,4 @@+import { aiFailureNotice } from '@mimik/core/capture/ai/errors';import { i18n } from '@mimik/core/env';import type { OverlayView } from '../../main/overlay';import { icon } from '../icons';@@ -77,11 +78,20 @@ export function controls(): void {});const modes = el('div', { id: 'modeBlock' }, modeLabel, modeRow);+ const aiHeadline = el('strong', {});+ const aiAction = el('span', {});+ const aiNotice = el(+ 'div',+ { id: 'aiNotice', role: 'status' },+ icon('alert', 13),+ el('p', {}, aiHeadline, ' ', aiAction),+ );+const secondary = el('button', { id: 'secondary', type: 'button' });const primary = el('button', { id: 'primary', type: 'button', className: 'primary' });const foot = el('div', { id: 'foot' }, secondary, primary);- document.body.append(head, body, modes, foot);+ document.body.append(head, body, modes, aiNotice, foot);const report = () => window.mimikOverlay.size(document.body.scrollWidth, document.body.scrollHeight);@@ -90,7 +100,7 @@ export function controls(): void {let shownMode = '';const render = (view: OverlayView) => {- const { state, step, busy, starting, mode, shortcuts } = view;+ const { state, step, busy, starting, mode, shortcuts, aiFailure } = view;const armed = state === 'armed';const recording = state === 'recording';const paused = state === 'paused';@@ -185,6 +195,13 @@ export function controls(): void {primary.disabled = starting || (busy && !armed);foot.hidden = collapsed;+ aiNotice.hidden = collapsed || armed || !aiFailure;+ if (aiFailure) {+ const notice = aiFailureNotice(aiFailure.reason, aiFailure.provider);+ aiHeadline.textContent = notice.headline;+ aiAction.textContent = notice.action;+ }+requestAnimationFrame(report);};diff --git a/packages/core/src/capture/ai/__tests__/errors.test.ts b/packages/core/src/capture/ai/__tests__/errors.test.tsindex 6136ca4..03d2fe0 100644--- a/packages/core/src/capture/ai/__tests__/errors.test.ts+++ b/packages/core/src/capture/ai/__tests__/errors.test.ts@@ -2,9 +2,7 @@ import { readFileSync } from 'node:fs';import { join } from 'node:path';import { APICallError, RetryError } from 'ai';import { describe, expect, it } from 'vitest';-import { aiActionKey } from '@/ui/sidepanel/lib/ai-action-key';-import { aiFailureKey } from '@/ui/sidepanel/lib/ai-failure-key';-import { type AiFailureReason, describeAiFailure } from '../errors';+import { type AiFailureReason, aiActionKey, aiFailureKey, describeAiFailure } from '../errors';const LOCALES = ['en', 'de', 'es', 'fr', 'pt-BR', 'zh-CN'];const REASONS: AiFailureReason[] = [diff --git a/packages/core/src/capture/ai/errors.ts b/packages/core/src/capture/ai/errors.tsindex 191e035..ebdf801 100644--- a/packages/core/src/capture/ai/errors.ts+++ b/packages/core/src/capture/ai/errors.ts@@ -1,3 +1,6 @@+import { i18n } from '@/core/env';+import { findProvider } from './models';+export type AiFailureReason =| 'rejected'| 'no-credits'@@ -43,3 +46,39 @@ export function describeAiFailure(err: unknown): AiFailure {if (status !== undefined) return { reason: 'network', status, message };return { reason: 'unknown', message };}++const AI_FAILURE_KEYS: Record<AiFailureReason, string> = {+ rejected: 'aiStatus.rejected',+ 'no-credits': 'aiStatus.noCredits',+ 'rate-limited': 'aiStatus.rateLimited',+ 'model-invalid': 'aiStatus.modelInvalid',+ quota: 'aiStatus.quota',+ network: 'aiStatus.network',+ unknown: 'aiStatus.unknown',+};++const AI_ACTION_KEYS: Record<AiFailureReason, string> = {+ rejected: 'aiStatus.actionCheckKey',+ 'no-credits': 'aiStatus.actionAddCredits',+ 'rate-limited': 'aiStatus.actionWait',+ 'model-invalid': 'aiStatus.actionPickModel',+ quota: 'aiStatus.actionPickModel',+ network: 'aiStatus.actionConnection',+ unknown: 'aiStatus.actionCheckKey',+};++export function aiFailureKey(reason: AiFailureReason | undefined): string {+ return (reason && AI_FAILURE_KEYS[reason]) ?? AI_FAILURE_KEYS.unknown;+}++export function aiActionKey(reason: AiFailureReason | undefined): string {+ return (reason && AI_ACTION_KEYS[reason]) ?? AI_ACTION_KEYS.unknown;+}++export function aiFailureNotice(+ reason: AiFailureReason | undefined,+ provider: string,+): { headline: string; action: string } {+ const label = findProvider(provider)?.label ?? provider;+ return { headline: i18n.t(aiFailureKey(reason), [label]), action: i18n.t(aiActionKey(reason)) };+}diff --git a/src/ui/sidepanel/AiStatus.tsx b/src/ui/sidepanel/AiStatus.tsxindex 42764ca..ea882bd 100644--- a/src/ui/sidepanel/AiStatus.tsx+++ b/src/ui/sidepanel/AiStatus.tsx@@ -1,9 +1,6 @@import { TriangleAlert } from 'lucide-react';-import { i18n } from '#imports';-import { findProvider } from '@/core/capture/ai/models';+import { aiFailureNotice } from '@/core/capture/ai/errors';import type { PanelAiUpdate } from '@/lib/port/types';-import { aiActionKey } from './lib/ai-action-key';-import { aiFailureKey } from './lib/ai-failure-key';interface AiStatusProps {update: PanelAiUpdate | null;@@ -12,14 +9,13 @@ interface AiStatusProps {export function AiStatus({ update }: AiStatusProps) {if (!update) return null;- const label = findProvider(update.provider)?.label ?? update.provider;+ const { headline, action } = aiFailureNotice(update.reason, update.provider);return (<div className="px-4 pt-2.5 flex items-start gap-2" role="status"><TriangleAlert size={13} className="shrink-0 mt-0.5 text-destructive" /><p className="text-[10px] leading-relaxed text-muted-foreground">- <span className="font-semibold text-foreground">{i18n.t(aiFailureKey(update.reason), [label])}</span>{' '}- {i18n.t(aiActionKey(update.reason))}+ <span className="font-semibold text-foreground">{headline}</span> {action}</p></div>);diff --git a/src/ui/sidepanel/lib/ai-action-key.ts b/src/ui/sidepanel/lib/ai-action-key.tsdeleted file mode 100644index 33c2440..0000000--- a/src/ui/sidepanel/lib/ai-action-key.ts+++ /dev/null@@ -1,16 +0,0 @@-import type { AiFailureReason } from '@/core/capture/ai/errors';--const AI_ACTION_KEYS: Record<AiFailureReason, string> = {- rejected: 'aiStatus.actionCheckKey',- 'no-credits': 'aiStatus.actionAddCredits',- 'rate-limited': 'aiStatus.actionWait',- 'model-invalid': 'aiStatus.actionPickModel',- quota: 'aiStatus.actionPickModel',- network: 'aiStatus.actionConnection',- unknown: 'aiStatus.actionCheckKey',-};--export function aiActionKey(reason: AiFailureReason | undefined): string {- const key = reason === undefined ? undefined : AI_ACTION_KEYS[reason];- return key ?? AI_ACTION_KEYS.unknown;-}diff --git a/src/ui/sidepanel/lib/ai-failure-key.ts b/src/ui/sidepanel/lib/ai-failure-key.tsdeleted file mode 100644index 71a3a18..0000000--- a/src/ui/sidepanel/lib/ai-failure-key.ts+++ /dev/null@@ -1,16 +0,0 @@-import type { AiFailureReason } from '@/core/capture/ai/errors';--const AI_FAILURE_KEYS: Record<AiFailureReason, string> = {- rejected: 'aiStatus.rejected',- 'no-credits': 'aiStatus.noCredits',- 'rate-limited': 'aiStatus.rateLimited',- 'model-invalid': 'aiStatus.modelInvalid',- quota: 'aiStatus.quota',- network: 'aiStatus.network',- unknown: 'aiStatus.unknown',-};--export function aiFailureKey(reason: AiFailureReason | undefined): string {- const key = reason === undefined ? undefined : AI_FAILURE_KEYS[reason];- return key ?? AI_FAILURE_KEYS.unknown;-}