Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots 📖

feat(ai): recognise custom model ids and retry guide meta as plain text

Luis Ortize268e9df8390
diff --git a/src/core/capture/ai/__tests__/meta.test.ts b/src/core/capture/ai/__tests__/meta.test.ts
index 4b47c2e..e2fbf41 100644
--- a/src/core/capture/ai/__tests__/meta.test.ts
+++ b/src/core/capture/ai/__tests__/meta.test.ts
@@ -1,9 +1,13 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
-const { generateObjectMock } = vi.hoisted(() => ({ generateObjectMock: vi.fn() }));
+const { generateObjectMock, generateTextMock } = vi.hoisted(() => ({
+ generateObjectMock: vi.fn(),
+ generateTextMock: vi.fn(),
+}));
vi.mock('ai', () => ({
generateObject: generateObjectMock,
+ generateText: generateTextMock,
jsonSchema: (schema: unknown) => schema,
}));
@@ -13,13 +17,16 @@ vi.mock('@/lib/browser-api', () => ({
localStorage: { get: vi.fn().mockResolvedValue({ aiLanguage: 'en' }) },
}));
-import { generateGuideMeta } from '../meta';
+import { generateGuideMeta, parseGuideMeta } from '../meta';
const steps = [{ description: 'Click Directory', url: 'https://admin.okta.com/users' }];
+const UNSUPPORTED = new Error('response_format json_schema is not supported by this model');
+
describe('generateGuideMeta', () => {
beforeEach(() => {
generateObjectMock.mockReset();
+ generateTextMock.mockReset();
});
it('returns both title and description from a well-formed response', async () => {
@@ -95,8 +102,91 @@ describe('generateGuideMeta', () => {
expect(generateObjectMock).not.toHaveBeenCalled();
});
- it('returns null when the request throws', async () => {
+ it('returns null when both the structured call and the text retry throw', async () => {
generateObjectMock.mockRejectedValue(new Error('rate limited'));
+ generateTextMock.mockRejectedValue(new Error('rate limited'));
expect(await generateGuideMeta(steps, 'openai', 'gpt-4o-mini', 'key')).toBeNull();
});
});
+
+describe('models without structured output support', () => {
+ beforeEach(() => {
+ generateObjectMock.mockReset();
+ generateTextMock.mockReset();
+ generateObjectMock.mockRejectedValue(UNSUPPORTED);
+ });
+
+ it('falls back to a plain text call and still yields a title', async () => {
+ generateTextMock.mockResolvedValue({
+ text: '{"title": "Check Recent Deaths on Wikipedia", "description": "Look up an entry."}',
+ });
+
+ const result = await generateGuideMeta(steps, 'openai', 'gpt-3.5-turbo', 'key');
+
+ expect(result).toEqual({
+ title: 'Check Recent Deaths on Wikipedia',
+ description: 'Look up an entry.',
+ });
+ });
+
+ it('asks the fallback call for bare JSON', async () => {
+ generateTextMock.mockResolvedValue({ text: '{"title": "T"}' });
+
+ await generateGuideMeta(steps, 'openai', 'gpt-3.5-turbo', 'key');
+
+ const { prompt } = generateTextMock.mock.calls[0][0];
+ expect(prompt).toContain('1. [https://admin.okta.com/users] Click Directory');
+ expect(prompt).toContain('"title"');
+ });
+
+ it('does not spend a second call when the structured call succeeds', async () => {
+ generateObjectMock.mockReset();
+ generateObjectMock.mockResolvedValue({ object: { title: 'T', description: 'D' } });
+
+ await generateGuideMeta(steps, 'openai', 'gpt-4o-mini', 'key');
+
+ expect(generateTextMock).not.toHaveBeenCalled();
+ });
+});
+
+describe('parseGuideMeta', () => {
+ it('reads a bare JSON object', () => {
+ expect(parseGuideMeta('{"title": "Okta Reset", "description": "Reset it."}')).toEqual({
+ title: 'Okta Reset',
+ description: 'Reset it.',
+ });
+ });
+
+ it('unwraps a fenced code block the model added anyway', () => {
+ expect(parseGuideMeta('```json\n{"title": "Okta Reset"}\n```')).toEqual({
+ title: 'Okta Reset',
+ description: undefined,
+ });
+ });
+
+ it('ignores prose wrapped around the object', () => {
+ expect(parseGuideMeta('Sure! Here you go:\n{"title": "Okta Reset"}\nHope that helps.')?.title).toBe('Okta Reset');
+ });
+
+ it('takes a bare line as the title when the model ignored the JSON request', () => {
+ expect(parseGuideMeta('Reset a User Password in Okta')?.title).toBe('Reset a User Password in Okta');
+ });
+
+ it('refuses a bare line that is really a lead-in to broken JSON', () => {
+ expect(parseGuideMeta('Here is the JSON: {"title": ')).toBeNull();
+ });
+
+ it('refuses a bare line too long to be a title', () => {
+ expect(parseGuideMeta('x'.repeat(120))).toBeNull();
+ });
+
+ it('truncates an over-long title from the fallback path too', () => {
+ const result = parseGuideMeta(JSON.stringify({ title: 'x'.repeat(90) }));
+ expect(result?.title.length).toBe(70);
+ expect(result?.title.endsWith('...')).toBe(true);
+ });
+
+ it('returns null for an empty response', () => {
+ expect(parseGuideMeta(' ')).toBeNull();
+ });
+});
diff --git a/src/core/capture/ai/__tests__/models.test.ts b/src/core/capture/ai/__tests__/models.test.ts
new file mode 100644
index 0000000..d023619
--- /dev/null
+++ b/src/core/capture/ai/__tests__/models.test.ts
@@ -0,0 +1,67 @@
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { describe, expect, it } from 'vitest';
+import { AI_PROVIDERS, CUSTOM_MODEL_VALUE, isCustomModel } from '../models';
+
+const LOCALES = ['en', 'de', 'es', 'fr', 'pt-BR'];
+
+const MODEL_KEYS = ['settings.model', 'settings.modelCustom'];
+
+function localeKeys(locale: string): Set<string> {
+ const keys = new Set<string>();
+ let section = '';
+ for (const line of readFileSync(join(process.cwd(), 'src/locales', `${locale}.yml`), 'utf8').split('\n')) {
+ const top = /^([\w-]+):/.exec(line);
+ if (top) {
+ section = top[1];
+ continue;
+ }
+ const nested = /^ {2}([\w-]+):/.exec(line);
+ if (nested && section) keys.add(`${section}.${nested[1]}`);
+ }
+ return keys;
+}
+
+describe('isCustomModel', () => {
+ it('treats a curated model as not custom', () => {
+ expect(isCustomModel('gpt-4o-mini', AI_PROVIDERS.openai)).toBe(false);
+ });
+
+ it('reopens a stored model the provider list does not carry', () => {
+ expect(isCustomModel('gpt-4o-2024-11-20', AI_PROVIDERS.openai)).toBe(true);
+ });
+
+ it('reads a model from another provider as custom', () => {
+ expect(isCustomModel('claude-3-5-haiku-20241022', AI_PROVIDERS.openai)).toBe(true);
+ });
+
+ it('falls back to the curated list when nothing is stored', () => {
+ expect(isCustomModel('', AI_PROVIDERS.openai)).toBe(false);
+ expect(isCustomModel(' ', AI_PROVIDERS.openai)).toBe(false);
+ });
+});
+
+describe('custom model sentinel', () => {
+ it('never collides with a real model id', () => {
+ for (const config of Object.values(AI_PROVIDERS)) {
+ expect(config.models.some((option) => option.id === CUSTOM_MODEL_VALUE)).toBe(false);
+ }
+ });
+
+ it('never renders as the empty value Radix rejects for an item', () => {
+ expect(CUSTOM_MODEL_VALUE.length).toBeGreaterThan(0);
+ });
+});
+
+describe('every provider default is selectable', () => {
+ it.each(Object.entries(AI_PROVIDERS))('%s lists its own default model', (_key, config) => {
+ expect(config.models.some((option) => option.id === config.defaultModel)).toBe(true);
+ });
+});
+
+describe('model picker copy', () => {
+ it.each(LOCALES)('%s carries every key the model picker renders', (locale) => {
+ const keys = localeKeys(locale);
+ for (const key of MODEL_KEYS) expect(keys).toContain(key);
+ });
+});
diff --git a/src/core/capture/ai/meta.ts b/src/core/capture/ai/meta.ts
index fd95138..7f683a0 100644
--- a/src/core/capture/ai/meta.ts
+++ b/src/core/capture/ai/meta.ts
@@ -1,7 +1,7 @@
-import { generateObject, jsonSchema } from 'ai';
+import { generateObject, generateText, jsonSchema } from 'ai';
import { localStorage } from '@/lib/browser-api';
import { logger } from '@/lib/logger';
-import { GUIDE_META_PROMPT, getLanguageSuffix } from './prompts';
+import { GUIDE_META_JSON_SUFFIX, GUIDE_META_PROMPT, getLanguageSuffix } from './prompts';
import { createModel } from './provider';
export interface GuideMeta {
@@ -9,6 +9,10 @@ export interface GuideMeta {
description?: string;
}
+const MAX_TITLE_LENGTH = 70;
+const MAX_BARE_TITLE_LENGTH = 100;
+const FENCE = /^```(?:json)?\s*|\s*```$/g;
+
const guideMetaSchema = jsonSchema<{ title: string; description?: string | null }>({
type: 'object',
properties: {
@@ -19,6 +23,36 @@ const guideMetaSchema = jsonSchema<{ title: string; description?: string | null
additionalProperties: false,
});
+function toGuideMeta(rawTitle: unknown, rawDescription: unknown): GuideMeta | null {
+ let title = typeof rawTitle === 'string' ? rawTitle.trim().replace(/^"|"$/g, '') : '';
+ if (!title) return null;
+ if (title.length > MAX_TITLE_LENGTH) title = `${title.slice(0, MAX_TITLE_LENGTH - 3)}...`;
+
+ const description =
+ typeof rawDescription === 'string' ? rawDescription.trim().replace(/^"|"$/g, '') || undefined : undefined;
+ return { title, description };
+}
+
+export function parseGuideMeta(raw: string): GuideMeta | null {
+ const text = raw.trim().replace(FENCE, '').trim();
+
+ const start = text.indexOf('{');
+ const end = text.lastIndexOf('}');
+ if (start !== -1 && end > start) {
+ try {
+ const parsed = JSON.parse(text.slice(start, end + 1)) as Record<string, unknown>;
+ const meta = toGuideMeta(parsed.title, parsed.description);
+ if (meta) return meta;
+ } catch {
+ logger.warn('Guide meta fallback returned malformed JSON');
+ }
+ }
+
+ const line = text.split('\n')[0]?.trim() ?? '';
+ if (!line || line.length > MAX_BARE_TITLE_LENGTH || line.includes('{')) return null;
+ return toGuideMeta(line, null);
+}
+
export async function generateGuideMeta(
steps: { description: string; url: string }[],
provider: string,
@@ -28,23 +62,30 @@ export async function generateGuideMeta(
if (steps.length === 0) return null;
const formatted = steps.map((s, i) => `${i + 1}. [${s.url}] ${s.description}`).join('\n');
+ const settings = await localStorage.get(['aiLanguage']);
+ const locale = (settings.aiLanguage as string) || 'en';
+ const prompt = GUIDE_META_PROMPT.replace('{{steps}}', formatted) + getLanguageSuffix(locale);
+ const aiModel = createModel(provider, model, apiKey);
try {
- const settings = await localStorage.get(['aiLanguage']);
- const locale = (settings.aiLanguage as string) || 'en';
const { object } = await generateObject({
- model: createModel(provider, model, apiKey),
+ model: aiModel,
schema: guideMetaSchema,
- prompt: GUIDE_META_PROMPT.replace('{{steps}}', formatted) + getLanguageSuffix(locale),
+ prompt,
maxOutputTokens: 200,
});
+ return toGuideMeta(object.title, object.description);
+ } catch (err) {
+ logger.warn('Structured guide meta failed, retrying as plain text', err);
+ }
- let title = object.title?.trim().replace(/^"|"$/g, '') ?? '';
- if (!title) return null;
- if (title.length > 70) title = `${title.slice(0, 67)}...`;
-
- const description = object.description?.trim().replace(/^"|"$/g, '') || undefined;
- return { title, description };
+ try {
+ const { text } = await generateText({
+ model: aiModel,
+ prompt: prompt + GUIDE_META_JSON_SUFFIX,
+ maxOutputTokens: 200,
+ });
+ return parseGuideMeta(text);
} catch (err) {
logger.error('Guide meta generation failed', err);
return null;
diff --git a/src/core/capture/ai/models.ts b/src/core/capture/ai/models.ts
index 9058efb..9c7685a 100644
--- a/src/core/capture/ai/models.ts
+++ b/src/core/capture/ai/models.ts
@@ -32,3 +32,10 @@ export const AI_PROVIDERS: Record<string, AIProviderConfig> = {
};
export type AIProviderKey = keyof typeof AI_PROVIDERS;
+
+export const CUSTOM_MODEL_VALUE = 'mimik-custom-model';
+
+export function isCustomModel(model: string, provider: AIProviderConfig): boolean {
+ const id = model.trim();
+ return id.length > 0 && !provider.models.some((option) => option.id === id);
+}
diff --git a/src/core/capture/ai/prompts.ts b/src/core/capture/ai/prompts.ts
index 47e4ebd..12077e4 100644
--- a/src/core/capture/ai/prompts.ts
+++ b/src/core/capture/ai/prompts.ts
@@ -30,6 +30,10 @@ Examples of good descriptions:
- "Reset a locked-out user's password from the Okta admin panel. For IT support staff."
- "Configure which Slack channels send desktop notifications, and set a do-not-disturb schedule."`;
+export const GUIDE_META_JSON_SUFFIX = `
+
+Reply with nothing but a JSON object shaped {"title": string, "description": string}. No code fence, no commentary.`;
+
export const REWRITE_PROMPT = `You are editing one span of text inside a browser workflow guide. The text describes a step a reader must perform, or summarises what the workflow accomplishes.
Selected text:
diff --git a/src/locales/de.yml b/src/locales/de.yml
index 83a4402..5f84c21 100644
--- a/src/locales/de.yml
+++ b/src/locales/de.yml
@@ -96,6 +96,7 @@ settings:
targetColor: "Markenfarbe"
title: Einstellungen
aiDescriptions: KI-Beschreibungen
+ modelCustom: Eigenes Modell
provider: Anbieter
model: Modell
apiKey: API-Schlüssel
diff --git a/src/locales/en.yml b/src/locales/en.yml
index 287f43d..f38167f 100644
--- a/src/locales/en.yml
+++ b/src/locales/en.yml
@@ -84,6 +84,7 @@ settings:
aiDescriptions: AI Descriptions
provider: Provider
model: Model
+ modelCustom: Custom model
apiKey: API Key
aiLanguage: Description Language
voiceNarration: Voice Narration
diff --git a/src/locales/es.yml b/src/locales/es.yml
index 154bccb..bbfaa04 100644
--- a/src/locales/es.yml
+++ b/src/locales/es.yml
@@ -84,6 +84,7 @@ settings:
aiDescriptions: Descripciones con IA
provider: Proveedor
model: Modelo
+ modelCustom: Modelo personalizado
apiKey: Clave API
aiLanguage: Idioma de descripciones
voiceNarration: "Narración por voz"
diff --git a/src/locales/fr.yml b/src/locales/fr.yml
index 88232bf..678b20b 100644
--- a/src/locales/fr.yml
+++ b/src/locales/fr.yml
@@ -84,6 +84,7 @@ settings:
aiDescriptions: Descriptions IA
provider: Fournisseur
model: "Modèle"
+ modelCustom: Modèle personnalisé
apiKey: "Clé API"
aiLanguage: Langue des descriptions
voiceNarration: "Narration vocale"
diff --git a/src/locales/pt-BR.yml b/src/locales/pt-BR.yml
index 9c2b78f..7a204b6 100644
--- a/src/locales/pt-BR.yml
+++ b/src/locales/pt-BR.yml
@@ -84,6 +84,7 @@ settings:
aiDescriptions: Descrições com IA
provider: Provedor
model: Modelo
+ modelCustom: Modelo personalizado
apiKey: Chave da API
aiLanguage: "Idioma das descrições"
voiceNarration: "Narração por voz"