Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

feat(desktop): say on the recording card why an AI description failed

Luis Ortiza82b0dc4ee4d
diff --git a/apps/desktop/scripts/check-overlay.ts b/apps/desktop/scripts/check-overlay.ts
index da5c552..dddaad2 100644
--- a/apps/desktop/scripts/check-overlay.ts
+++ b/apps/desktop/scripts/check-overlay.ts
@@ -274,6 +274,19 @@ app.whenReady().then(async () => {
shortTitle.height === veil.height && longTitle.height === veil.height && longTitle.preview < shortTitle.preview,
`${veil.height} px while capturing, ${shortTitle.height} px with a short title, ${longTitle.height} px with a long one, preview ${shortTitle.preview} β†’ ${longTitle.preview} px`,
);
+ overlay.setAiFailure({ reason: 'rejected', provider: 'openai' });
+ await settle();
+ const failed = await card(
+ "shown: !document.querySelector('#aiNotice').hidden, text: document.querySelector('#aiNotice').textContent",
+ );
+ overlay.setAiFailure(null);
+ await settle();
+ const cleared = await card("shown: !document.querySelector('#aiNotice').hidden");
+ check(
+ 'a failed AI description says why, in the extension words',
+ failed.shown === true && String(failed.text).includes('OpenAI') && cleared.shown === false,
+ `notice: ${failed.text}; after clearing shown: ${cleared.shown}`,
+ );
const whenIdle = await windowWithHash('controls')?.webContents.executeJavaScript(
"document.querySelector('#primary').disabled",
);
diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts
index 3e45736..dabbb07 100644
--- a/apps/desktop/src/main/index.ts
+++ b/apps/desktop/src/main/index.ts
@@ -6,7 +6,7 @@ import { focusedWindow } from './capture/focused-window';
import { DesktopRecorder, frameFor } from './capture/recorder';
import { registerScreenshotProtocol, SCREENSHOT_SCHEME, sweepScreenshots } from './capture/screenshot-store';
import { type CaptureMode, type CaptureSettings, loadSettings, saveSettings } from './capture/settings';
-import { CaptureOverlay, type OverlayCommand, type OverlayStep } from './overlay';
+import { CaptureOverlay, type OverlayAiFailure, type OverlayCommand, type OverlayStep } from './overlay';
import { bindShortcuts, type ShortcutName, shortcutMap, unbindShortcuts } from './shortcuts';
import { checkForUpdates } from './updater';
@@ -248,6 +248,7 @@ async function onOverlayCommand(command: OverlayCommand): Promise<void> {
}
if (command === 'start' && !guideId) {
steps = [];
+ overlay?.setAiFailure(null);
try {
guideId = await ask<string>(mainWindow?.webContents ?? null, 'mimik:capture:startGuide');
} catch (error) {
@@ -374,16 +375,20 @@ if (!app.requestSingleInstanceLock()) {
applyShortcuts();
return captureSettings;
});
- ipcMain.on('mimik:capture:described', (_event, stepId: string, description: string | null) => {
- const step = steps.find((candidate) => candidate.id === stepId);
- if (!step) return;
- if (description) {
- step.title = description;
- step.source = 'ai';
- }
- step.pending = false;
- if (step === steps.at(-1)) overlay?.showStep(step);
- });
+ ipcMain.on(
+ 'mimik:capture:described',
+ (_event, stepId: string, description: string | null, failure: OverlayAiFailure | null) => {
+ if (failure) overlay?.setAiFailure(failure);
+ const step = steps.find((candidate) => candidate.id === stepId);
+ if (!step) return;
+ if (description) {
+ step.title = description;
+ step.source = 'ai';
+ }
+ step.pending = false;
+ if (step === steps.at(-1)) overlay?.showStep(step);
+ },
+ );
ipcMain.handle('mimik:capture:region', () => overlay?.region);
ipcMain.handle('mimik:capture:edit', () => {
enterCapture();
diff --git a/apps/desktop/src/main/overlay/index.ts b/apps/desktop/src/main/overlay/index.ts
index ad07ce0..5b63a96 100644
--- a/apps/desktop/src/main/overlay/index.ts
+++ b/apps/desktop/src/main/overlay/index.ts
@@ -1,4 +1,5 @@
import { join } from 'node:path';
+import type { AiFailureReason } from '@mimik/core/capture/ai/errors';
import { BrowserWindow, ipcMain, screen } from 'electron';
import { clampToDisplays, loadRegion, type Region, saveRegion } from '../capture/region';
import type { CaptureMode } from '../capture/settings';
@@ -33,6 +34,11 @@ export interface OverlayShortcuts {
capture: string | null;
}
+export interface OverlayAiFailure {
+ reason: AiFailureReason;
+ provider: string;
+}
+
export interface OverlayView {
state: OverlayState;
region: Region;
@@ -41,6 +47,7 @@ export interface OverlayView {
busy: boolean;
starting: boolean;
shortcuts: OverlayShortcuts;
+ aiFailure: OverlayAiFailure | null;
}
export interface OverlayOptions {
@@ -112,6 +119,7 @@ export class CaptureOverlay {
private step: OverlayStep | null = null;
private size = { width: CONTROLS.width, height: CONTROLS.height };
private busy = false;
+ private aiFailure: OverlayAiFailure | null = null;
private editingFrom: OverlayState = 'hidden';
private modeBeforeEdit: CaptureMode | null = null;
private hiding: { shown: BrowserWindow[]; ready: Promise<unknown>; users: number } | null = null;
@@ -157,6 +165,7 @@ export class CaptureOverlay {
busy: this.busy,
starting: this.starting,
shortcuts: this.options.shortcuts?.() ?? NO_SHORTCUTS,
+ aiFailure: this.aiFailure,
};
}
@@ -183,6 +192,11 @@ export class CaptureOverlay {
this.broadcast();
}
+ setAiFailure(failure: OverlayAiFailure | null): void {
+ this.aiFailure = failure;
+ this.broadcast();
+ }
+
setBusy(busy: boolean): void {
if (this.busy === busy) return;
this.busy = busy;
diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts
index 5506a5b..3a3c6c6 100644
--- a/apps/desktop/src/preload/index.ts
+++ b/apps/desktop/src/preload/index.ts
@@ -1,5 +1,6 @@
import { contextBridge, ipcRenderer } from 'electron';
import type { CaptureSettings } from '../main/capture/settings';
+import type { OverlayAiFailure } from '../main/overlay';
interface Region {
x: number;
@@ -14,8 +15,8 @@ const api = {
region: (): Promise<Region> => ipcRenderer.invoke('mimik:capture:region'),
edit: (): Promise<void> => ipcRenderer.invoke('mimik:capture:edit'),
arm: (): Promise<void> => ipcRenderer.invoke('mimik:capture:arm'),
- described: (stepId: string, description: string | null): void =>
- ipcRenderer.send('mimik:capture:described', stepId, description),
+ described: (stepId: string, description: string | null, failure: OverlayAiFailure | null): void =>
+ ipcRenderer.send('mimik:capture:described', stepId, description, failure),
settings: {
get: (): Promise<CaptureSettings> => ipcRenderer.invoke('mimik:capture:settings:get'),
set: (patch: Partial<CaptureSettings>): Promise<CaptureSettings> =>
diff --git a/apps/desktop/src/renderer/ai/describe-step.ts b/apps/desktop/src/renderer/ai/describe-step.ts
index 8e448dc..8d424b3 100644
--- a/apps/desktop/src/renderer/ai/describe-step.ts
+++ b/apps/desktop/src/renderer/ai/describe-step.ts
@@ -1,20 +1,29 @@
import { getAIDescription } from '@mimik/core/capture/ai/description';
+import { describeAiFailure } from '@mimik/core/capture/ai/errors';
import { serializeScreenContext } from '@mimik/core/capture/ai/screen-context';
import type { ElementMeta } from '@mimik/core/guides/types';
+import { logger } from '@mimik/core/logger';
+import type { OverlayAiFailure } from '../../main/overlay';
import { credentials } from './credentials';
-export async function describeStep(action: string, meta: ElementMeta): Promise<string | null> {
+export async function describeStep(
+ action: string,
+ meta: ElementMeta,
+): Promise<{ text: string | null; failure: OverlayAiFailure | null }> {
const keys = await credentials();
- if (!keys) return null;
+ if (!keys) return { text: null, failure: null };
try {
- return await getAIDescription(
+ const text = await getAIDescription(
serializeScreenContext(action, meta),
keys.provider,
keys.model,
keys.apiKey,
keys.baseUrl,
);
- } catch {
- return null;
+ return { text: text || null, failure: null };
+ } catch (err) {
+ const failure = describeAiFailure(err);
+ logger.error(`AI description failed (${failure.reason}${failure.status ? ` ${failure.status}` : ''})`, err);
+ return { text: null, failure: { reason: failure.reason, provider: keys.provider } };
}
}
diff --git a/apps/desktop/src/renderer/capture-sink.ts b/apps/desktop/src/renderer/capture-sink.ts
index 801c1c6..ef76658 100644
--- a/apps/desktop/src/renderer/capture-sink.ts
+++ b/apps/desktop/src/renderer/capture-sink.ts
@@ -86,10 +86,10 @@ export class DesktopCaptureSink implements CaptureSink {
await addStepToGuide(data.guideId, stepId);
if (pending) {
- void describeStep(data.action, meta).then(async (written) => {
- if (written) await updateStepDescription(stepId, written, 'ai');
+ void describeStep(data.action, meta).then(async ({ text, failure }) => {
+ if (text) await updateStepDescription(stepId, text, 'ai');
await clearStepAiPending(stepId);
- window.mimik.capture.described(stepId, written);
+ window.mimik.capture.described(stepId, text, failure);
});
}
diff --git a/apps/desktop/src/renderer/icons.ts b/apps/desktop/src/renderer/icons.ts
index 71f5e25..cdad930 100644
--- a/apps/desktop/src/renderer/icons.ts
+++ b/apps/desktop/src/renderer/icons.ts
@@ -11,6 +11,7 @@ import {
Pause,
Play,
Trash2,
+ TriangleAlert,
Video,
} from 'lucide';
@@ -27,6 +28,7 @@ const ICONS = {
monitor: Monitor,
area: Crop,
keyboard: Keyboard,
+ alert: TriangleAlert,
};
export type IconName = keyof typeof ICONS;
diff --git a/apps/desktop/src/renderer/overlay.css b/apps/desktop/src/renderer/overlay.css
index 7d83d52..c9d9c2b 100644
--- a/apps/desktop/src/renderer/overlay.css
+++ b/apps/desktop/src/renderer/overlay.css
@@ -484,6 +484,31 @@ body.controls #skeleton .meta span {
}
}
+body.controls #aiNotice {
+ display: flex;
+ align-items: flex-start;
+ gap: 7px;
+ padding: 10px 14px 0;
+ font-size: 11px;
+ line-height: 1.45;
+ color: var(--muted);
+}
+
+body.controls #aiNotice svg {
+ flex-shrink: 0;
+ margin-top: 1px;
+ color: #dc2626;
+}
+
+body.controls #aiNotice p {
+ margin: 0;
+}
+
+body.controls #aiNotice strong {
+ font-weight: 600;
+ color: var(--deep);
+}
+
body.controls #stepMeta,
body.controls #skeleton .meta {
display: flex;
diff --git a/apps/desktop/src/renderer/overlay/controls.ts b/apps/desktop/src/renderer/overlay/controls.ts
index 5c99982..e2efe26 100644
--- a/apps/desktop/src/renderer/overlay/controls.ts
+++ b/apps/desktop/src/renderer/overlay/controls.ts
@@ -1,3 +1,4 @@
+import { aiFailureNotice } from '@mimik/core/capture/ai/errors';
import { i18n } from '@mimik/core/env';
import type { OverlayView } from '../../main/overlay';
import { icon } from '../icons';
@@ -77,11 +78,20 @@ export function controls(): void {
});
const modes = el('div', { id: 'modeBlock' }, modeLabel, modeRow);
+ const aiHeadline = el('strong', {});
+ const aiAction = el('span', {});
+ const aiNotice = el(
+ 'div',
+ { id: 'aiNotice', role: 'status' },
+ icon('alert', 13),
+ el('p', {}, aiHeadline, ' ', aiAction),
+ );
+
const secondary = el('button', { id: 'secondary', type: 'button' });
const primary = el('button', { id: 'primary', type: 'button', className: 'primary' });
const foot = el('div', { id: 'foot' }, secondary, primary);
- document.body.append(head, body, modes, foot);
+ document.body.append(head, body, modes, aiNotice, foot);
const report = () => window.mimikOverlay.size(document.body.scrollWidth, document.body.scrollHeight);
@@ -90,7 +100,7 @@ export function controls(): void {
let shownMode = '';
const render = (view: OverlayView) => {
- const { state, step, busy, starting, mode, shortcuts } = view;
+ const { state, step, busy, starting, mode, shortcuts, aiFailure } = view;
const armed = state === 'armed';
const recording = state === 'recording';
const paused = state === 'paused';
@@ -185,6 +195,13 @@ export function controls(): void {
primary.disabled = starting || (busy && !armed);
foot.hidden = collapsed;
+ aiNotice.hidden = collapsed || armed || !aiFailure;
+ if (aiFailure) {
+ const notice = aiFailureNotice(aiFailure.reason, aiFailure.provider);
+ aiHeadline.textContent = notice.headline;
+ aiAction.textContent = notice.action;
+ }
+
requestAnimationFrame(report);
};
diff --git a/packages/core/src/capture/ai/__tests__/errors.test.ts b/packages/core/src/capture/ai/__tests__/errors.test.ts
index 6136ca4..03d2fe0 100644
--- a/packages/core/src/capture/ai/__tests__/errors.test.ts
+++ b/packages/core/src/capture/ai/__tests__/errors.test.ts
@@ -2,9 +2,7 @@ import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { APICallError, RetryError } from 'ai';
import { describe, expect, it } from 'vitest';
-import { aiActionKey } from '@/ui/sidepanel/lib/ai-action-key';
-import { aiFailureKey } from '@/ui/sidepanel/lib/ai-failure-key';
-import { type AiFailureReason, describeAiFailure } from '../errors';
+import { type AiFailureReason, aiActionKey, aiFailureKey, describeAiFailure } from '../errors';
const LOCALES = ['en', 'de', 'es', 'fr', 'pt-BR', 'zh-CN'];
const REASONS: AiFailureReason[] = [
diff --git a/packages/core/src/capture/ai/errors.ts b/packages/core/src/capture/ai/errors.ts
index 191e035..ebdf801 100644
--- a/packages/core/src/capture/ai/errors.ts
+++ b/packages/core/src/capture/ai/errors.ts
@@ -1,3 +1,6 @@
+import { i18n } from '@/core/env';
+import { findProvider } from './models';
+
export type AiFailureReason =
| 'rejected'
| 'no-credits'
@@ -43,3 +46,39 @@ export function describeAiFailure(err: unknown): AiFailure {
if (status !== undefined) return { reason: 'network', status, message };
return { reason: 'unknown', message };
}
+
+const AI_FAILURE_KEYS: Record<AiFailureReason, string> = {
+ rejected: 'aiStatus.rejected',
+ 'no-credits': 'aiStatus.noCredits',
+ 'rate-limited': 'aiStatus.rateLimited',
+ 'model-invalid': 'aiStatus.modelInvalid',
+ quota: 'aiStatus.quota',
+ network: 'aiStatus.network',
+ unknown: 'aiStatus.unknown',
+};
+
+const AI_ACTION_KEYS: Record<AiFailureReason, string> = {
+ rejected: 'aiStatus.actionCheckKey',
+ 'no-credits': 'aiStatus.actionAddCredits',
+ 'rate-limited': 'aiStatus.actionWait',
+ 'model-invalid': 'aiStatus.actionPickModel',
+ quota: 'aiStatus.actionPickModel',
+ network: 'aiStatus.actionConnection',
+ unknown: 'aiStatus.actionCheckKey',
+};
+
+export function aiFailureKey(reason: AiFailureReason | undefined): string {
+ return (reason && AI_FAILURE_KEYS[reason]) ?? AI_FAILURE_KEYS.unknown;
+}
+
+export function aiActionKey(reason: AiFailureReason | undefined): string {
+ return (reason && AI_ACTION_KEYS[reason]) ?? AI_ACTION_KEYS.unknown;
+}
+
+export function aiFailureNotice(
+ reason: AiFailureReason | undefined,
+ provider: string,
+): { headline: string; action: string } {
+ const label = findProvider(provider)?.label ?? provider;
+ return { headline: i18n.t(aiFailureKey(reason), [label]), action: i18n.t(aiActionKey(reason)) };
+}
diff --git a/src/ui/sidepanel/AiStatus.tsx b/src/ui/sidepanel/AiStatus.tsx
index 42764ca..ea882bd 100644
--- a/src/ui/sidepanel/AiStatus.tsx
+++ b/src/ui/sidepanel/AiStatus.tsx
@@ -1,9 +1,6 @@
import { TriangleAlert } from 'lucide-react';
-import { i18n } from '#imports';
-import { findProvider } from '@/core/capture/ai/models';
+import { aiFailureNotice } from '@/core/capture/ai/errors';
import type { PanelAiUpdate } from '@/lib/port/types';
-import { aiActionKey } from './lib/ai-action-key';
-import { aiFailureKey } from './lib/ai-failure-key';
interface AiStatusProps {
update: PanelAiUpdate | null;
@@ -12,14 +9,13 @@ interface AiStatusProps {
export function AiStatus({ update }: AiStatusProps) {
if (!update) return null;
- const label = findProvider(update.provider)?.label ?? update.provider;
+ const { headline, action } = aiFailureNotice(update.reason, update.provider);
return (
<div className="px-4 pt-2.5 flex items-start gap-2" role="status">
<TriangleAlert size={13} className="shrink-0 mt-0.5 text-destructive" />
<p className="text-[10px] leading-relaxed text-muted-foreground">
- <span className="font-semibold text-foreground">{i18n.t(aiFailureKey(update.reason), [label])}</span>{' '}
- {i18n.t(aiActionKey(update.reason))}
+ <span className="font-semibold text-foreground">{headline}</span> {action}
</p>
</div>
);
diff --git a/src/ui/sidepanel/lib/ai-action-key.ts b/src/ui/sidepanel/lib/ai-action-key.ts
deleted file mode 100644
index 33c2440..0000000
--- a/src/ui/sidepanel/lib/ai-action-key.ts
+++ /dev/null
@@ -1,16 +0,0 @@
-import type { AiFailureReason } from '@/core/capture/ai/errors';
-
-const AI_ACTION_KEYS: Record<AiFailureReason, string> = {
- rejected: 'aiStatus.actionCheckKey',
- 'no-credits': 'aiStatus.actionAddCredits',
- 'rate-limited': 'aiStatus.actionWait',
- 'model-invalid': 'aiStatus.actionPickModel',
- quota: 'aiStatus.actionPickModel',
- network: 'aiStatus.actionConnection',
- unknown: 'aiStatus.actionCheckKey',
-};
-
-export function aiActionKey(reason: AiFailureReason | undefined): string {
- const key = reason === undefined ? undefined : AI_ACTION_KEYS[reason];
- return key ?? AI_ACTION_KEYS.unknown;
-}
diff --git a/src/ui/sidepanel/lib/ai-failure-key.ts b/src/ui/sidepanel/lib/ai-failure-key.ts
deleted file mode 100644
index 71a3a18..0000000
--- a/src/ui/sidepanel/lib/ai-failure-key.ts
+++ /dev/null
@@ -1,16 +0,0 @@
-import type { AiFailureReason } from '@/core/capture/ai/errors';
-
-const AI_FAILURE_KEYS: Record<AiFailureReason, string> = {
- rejected: 'aiStatus.rejected',
- 'no-credits': 'aiStatus.noCredits',
- 'rate-limited': 'aiStatus.rateLimited',
- 'model-invalid': 'aiStatus.modelInvalid',
- quota: 'aiStatus.quota',
- network: 'aiStatus.network',
- unknown: 'aiStatus.unknown',
-};
-
-export function aiFailureKey(reason: AiFailureReason | undefined): string {
- const key = reason === undefined ? undefined : AI_FAILURE_KEYS[reason];
- return key ?? AI_FAILURE_KEYS.unknown;
-}