Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

feat(desktop): write step descriptions and guide names with the user's own key

Luis Ortiz281c0862b11c
diff --git a/AGENTS.md b/AGENTS.md
index d91389e..e40332f 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -618,6 +618,21 @@ rather than the placeholder the extension fills in with AI. Desktop has no AI ti
comes from the recorded application, or a generic one where no application was identified. Without
it the guide screen waits forever on a title that is never written.
+Descriptions and the guide's name are written by the user's own provider key when there is one, and
+by rule when there is not. `getAIDescription` takes a serialised context string rather than a
+`DOMContext`, because the desktop has no DOM to hand it: `serializeScreenContext` writes the same
+shape of thing from the application, the window title, the control's role and name, and the value,
+which is what UIAutomation knows. No screenshot is ever sent.
+
+A step is written with its heuristic description immediately and `aiPending` set, then rewritten
+when the model answers. The flag is cleared **whichever way that goes** β€” a miss, a failure and a
+missing key all clear it β€” because a pending flag that only clears on success is the same trap as a
+title placeholder that only resolves with AI: without a key it stays there forever.
+
+Stopping a recording names the guide twice. The application name lands first so the view never opens
+on a placeholder, and `generateGuideMeta` replaces it if a key is configured. Ordering it that way
+means the guide is always named, and the AI title is an improvement rather than a prerequisite.
+
Step descriptions are only as good as the element lookup. With one, `buildFallbackDescription` gets
a role and an accessible name and writes the same wording it writes for the extension. Without one
it has nothing but the action, and every step in the guide reads the same β€” which is what a
diff --git a/apps/desktop/src/renderer/ai.ts b/apps/desktop/src/renderer/ai.ts
new file mode 100644
index 0000000..410cc41
--- /dev/null
+++ b/apps/desktop/src/renderer/ai.ts
@@ -0,0 +1,59 @@
+import { getAIDescription } from '@mimik/core/capture/ai/description';
+import { resolveAiKey } from '@mimik/core/capture/ai/keys';
+import { type GuideMeta, generateGuideMeta } from '@mimik/core/capture/ai/meta';
+import { AI_PROVIDERS } from '@mimik/core/capture/ai/models';
+import { serializeScreenContext } from '@mimik/core/capture/ai/screen-context';
+import { localStorage } from '@mimik/core/env';
+import type { ElementMeta } from '@mimik/core/guides/types';
+
+const AI_SETTINGS = ['aiApiKeys', 'aiApiKey', 'aiProvider', 'aiModel', 'aiBaseUrl'] as const;
+
+interface Credentials {
+ provider: string;
+ model: string;
+ apiKey: string;
+ baseUrl?: string;
+}
+
+async function credentials(): Promise<Credentials | null> {
+ const stored = await localStorage.get(AI_SETTINGS);
+ const { provider, apiKey } = resolveAiKey(stored);
+ if (!apiKey) return null;
+ return {
+ provider,
+ apiKey,
+ model: (stored.aiModel as string) || AI_PROVIDERS[provider].defaultModel,
+ baseUrl: (stored.aiBaseUrl as string) || undefined,
+ };
+}
+
+export async function describeStep(action: string, meta: ElementMeta): Promise<string | null> {
+ const keys = await credentials();
+ if (!keys) return null;
+ try {
+ return await getAIDescription(
+ serializeScreenContext(action, meta),
+ keys.provider,
+ keys.model,
+ keys.apiKey,
+ keys.baseUrl,
+ );
+ } catch {
+ return null;
+ }
+}
+
+export async function nameGuide(steps: { description: string }[]): Promise<GuideMeta | null> {
+ const keys = await credentials();
+ if (!keys) return null;
+ const described = steps
+ .filter((step) => step.description)
+ .map((step) => ({ description: step.description, url: '' }));
+ if (described.length === 0) return null;
+ const trimmed = described.length > 15 ? [...described.slice(0, 10), ...described.slice(-5)] : described;
+ try {
+ return await generateGuideMeta(trimmed, keys.provider, keys.model, keys.apiKey, keys.baseUrl);
+ } catch {
+ return null;
+ }
+}
diff --git a/apps/desktop/src/renderer/capture-host.ts b/apps/desktop/src/renderer/capture-host.ts
index e01e66b..059e485 100644
--- a/apps/desktop/src/renderer/capture-host.ts
+++ b/apps/desktop/src/renderer/capture-host.ts
@@ -4,8 +4,10 @@ import {
allScreenshotIds,
getStepsForGuide,
permanentlyDeleteGuide,
+ updateGuideDescription,
updateGuideTitle,
} from '@mimik/core/guides/service';
+import { nameGuide } from './ai';
import { DesktopCaptureSink } from './capture-sink';
const sink = new DesktopCaptureSink();
@@ -20,6 +22,10 @@ window.mimik.onRequest('mimik:capture:finishGuide', async (payload) => {
const steps = await getStepsForGuide(guideId);
const app = steps.find((step) => step.app?.name)?.app?.name;
await updateGuideTitle(guideId, app ? i18n.t('desktop.guideInApp', [app]) : i18n.t('desktop.newGuide'));
+
+ const meta = await nameGuide(steps);
+ if (meta?.title) await updateGuideTitle(guideId, meta.title);
+ if (meta?.description) await updateGuideDescription(guideId, meta.description);
return true;
});
diff --git a/apps/desktop/src/renderer/capture-sink.ts b/apps/desktop/src/renderer/capture-sink.ts
index 2c2806f..b478b90 100644
--- a/apps/desktop/src/renderer/capture-sink.ts
+++ b/apps/desktop/src/renderer/capture-sink.ts
@@ -11,6 +11,7 @@ import { buildFallbackDescription } from '@mimik/core/capture/step-description';
import { localStorage } from '@mimik/core/env';
import {
addStepToGuide,
+ clearStepAiPending,
createGuide,
createStep,
getStepsForGuide,
@@ -20,6 +21,7 @@ import {
import type { StoredScreenshot } from '@mimik/core/guides/types';
import type { CursorMark } from '@mimik/core/screenshot/types';
import { DEFAULT_TARGET_COLOR } from '@mimik/core/screenshot/types';
+import { describeStep } from './ai';
export type DesktopCaptureStepData = CaptureStepData & { cursor?: CursorMark; inputValue?: string };
@@ -79,10 +81,16 @@ export class DesktopCaptureSink implements CaptureSink {
screenshotId,
elementMeta: meta,
descriptionSource: 'heuristic',
+ aiPending: true,
...(data.inputValue === undefined ? {} : { inputValue: data.inputValue }),
});
await addStepToGuide(data.guideId, stepId);
+ void describeStep(data.action, meta).then(async (written) => {
+ if (written) await updateStepDescription(stepId, written);
+ await clearStepAiPending(stepId);
+ });
+
return { stepId, title: description };
}
diff --git a/packages/core/src/capture/ai/__tests__/screen-context.test.ts b/packages/core/src/capture/ai/__tests__/screen-context.test.ts
new file mode 100644
index 0000000..8934a52
--- /dev/null
+++ b/packages/core/src/capture/ai/__tests__/screen-context.test.ts
@@ -0,0 +1,49 @@
+import { describe, expect, it } from 'vitest';
+import type { ElementMeta } from '@/core/guides/types';
+import { serializeScreenContext } from '../screen-context';
+
+const meta = (over: Partial<ElementMeta> = {}): ElementMeta => ({
+ source: 'uia',
+ textContent: null,
+ ariaLabel: null,
+ placeholder: null,
+ altText: null,
+ name: null,
+ role: null,
+ rect: { x: 0, y: 0, width: 1, height: 1 },
+ devicePixelRatio: 1,
+ ...over,
+});
+
+describe('serializeScreenContext', () => {
+ it('names the application, window and control', () => {
+ const text = serializeScreenContext(
+ 'click',
+ meta({
+ role: 'button',
+ ariaLabel: 'Save',
+ app: { name: 'Notepad' },
+ window: { title: 'Untitled' },
+ }),
+ );
+ expect(text).toContain('Application: Notepad');
+ expect(text).toContain('Window: "Untitled"');
+ expect(text).toContain('β†’ Target: button "Save" (click)');
+ });
+
+ it('carries the typed value and says a key press is one', () => {
+ expect(serializeScreenContext('input', meta({ role: 'textbox', textContent: 'hello' }))).toContain(
+ 'Value: "hello"',
+ );
+ const key = serializeScreenContext('keydown:Ctrl+S', meta({ role: 'document' }));
+ expect(key).toContain('(key press)');
+ expect(key).toContain('Key: Ctrl+S');
+ });
+
+ it('says what it does not know rather than inventing it', () => {
+ const text = serializeScreenContext('click', meta());
+ expect(text).toContain('Application: unknown');
+ expect(text).toContain('β†’ Target: control (click)');
+ expect(text).not.toContain('Value:');
+ });
+});
diff --git a/packages/core/src/capture/ai/description.ts b/packages/core/src/capture/ai/description.ts
index 063148d..2ef7387 100644
--- a/packages/core/src/capture/ai/description.ts
+++ b/packages/core/src/capture/ai/description.ts
@@ -1,12 +1,10 @@
import { generateText } from 'ai';
import { localStorage } from '@/core/env';
-import type { DOMContext } from '../dom/context';
-import { serializeDOMContext } from '../dom/context';
import { getLanguageSuffix, STEP_DESCRIPTION_PROMPT } from './prompts';
import { createModel } from './provider';
export async function getAIDescription(
- domContext: DOMContext,
+ context: string,
provider: string,
model: string,
apiKey: string,
@@ -16,7 +14,7 @@ export async function getAIDescription(
const locale = (settings.aiLanguage as string) || 'en';
const { text } = await generateText({
model: createModel(provider, model, apiKey, baseUrl),
- prompt: STEP_DESCRIPTION_PROMPT.replace('{{context}}', serializeDOMContext(domContext)) + getLanguageSuffix(locale),
+ prompt: STEP_DESCRIPTION_PROMPT.replace('{{context}}', context) + getLanguageSuffix(locale),
maxOutputTokens: 50,
});
return text.trim().replace(/^"|"$/g, '') || null;
diff --git a/packages/core/src/capture/ai/screen-context.ts b/packages/core/src/capture/ai/screen-context.ts
new file mode 100644
index 0000000..787a45a
--- /dev/null
+++ b/packages/core/src/capture/ai/screen-context.ts
@@ -0,0 +1,18 @@
+import type { ElementMeta } from '@/core/guides/types';
+
+export function serializeScreenContext(action: string, meta: ElementMeta): string {
+ const lines: string[] = [];
+
+ lines.push(`Application: ${meta.app?.name ?? 'unknown'}`);
+ if (meta.window?.title) lines.push(`Window: "${meta.window.title}"`);
+
+ const name = meta.ariaLabel ?? meta.name;
+ const control = [meta.role ?? 'control', name ? `"${name}"` : null].filter(Boolean).join(' ');
+ lines.push(`β†’ Target: ${control} (${action.startsWith('keydown:') ? 'key press' : action})`);
+
+ if (meta.textContent) lines.push(`Value: "${meta.textContent.slice(0, 120)}"`);
+ if (meta.altText) lines.push(`Help: "${meta.altText}"`);
+ if (action.startsWith('keydown:')) lines.push(`Key: ${action.slice(8)}`);
+
+ return lines.join('\n');
+}
diff --git a/src/entrypoints/background/ai-description.ts b/src/entrypoints/background/ai-description.ts
index 542d2d9..c66efe8 100644
--- a/src/entrypoints/background/ai-description.ts
+++ b/src/entrypoints/background/ai-description.ts
@@ -3,7 +3,7 @@ import { getAIDescription } from '@/core/capture/ai/description';
import { describeAiFailure } from '@/core/capture/ai/errors';
import { resolveAiKey } from '@/core/capture/ai/keys';
import { AI_PROVIDERS } from '@/core/capture/ai/models';
-import type { DOMContext } from '@/core/capture/dom/context';
+import { type DOMContext, serializeDOMContext } from '@/core/capture/dom/context';
import { localStorage } from '@/lib/browser-api';
import { broadcastAiToPanel } from '@/lib/port';
@@ -15,7 +15,7 @@ export async function generateAiDescription(domContext: DOMContext): Promise<str
const model = (settings.aiModel as string) || AI_PROVIDERS[provider].defaultModel;
try {
const description = await getAIDescription(
- domContext,
+ serializeDOMContext(domContext),
provider,
model,
apiKey,