🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots 📖
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
feat(ai): recognise custom model ids and retry guide meta as plain text
Luis Ortize268e9df8390
diff --git a/src/core/capture/ai/__tests__/meta.test.ts b/src/core/capture/ai/__tests__/meta.test.tsindex 4b47c2e..e2fbf41 100644--- a/src/core/capture/ai/__tests__/meta.test.ts+++ b/src/core/capture/ai/__tests__/meta.test.ts@@ -1,9 +1,13 @@import { beforeEach, describe, expect, it, vi } from 'vitest';-const { generateObjectMock } = vi.hoisted(() => ({ generateObjectMock: vi.fn() }));+const { generateObjectMock, generateTextMock } = vi.hoisted(() => ({+ generateObjectMock: vi.fn(),+ generateTextMock: vi.fn(),+}));vi.mock('ai', () => ({generateObject: generateObjectMock,+ generateText: generateTextMock,jsonSchema: (schema: unknown) => schema,}));@@ -13,13 +17,16 @@ vi.mock('@/lib/browser-api', () => ({localStorage: { get: vi.fn().mockResolvedValue({ aiLanguage: 'en' }) },}));-import { generateGuideMeta } from '../meta';+import { generateGuideMeta, parseGuideMeta } from '../meta';const steps = [{ description: 'Click Directory', url: 'https://admin.okta.com/users' }];+const UNSUPPORTED = new Error('response_format json_schema is not supported by this model');+describe('generateGuideMeta', () => {beforeEach(() => {generateObjectMock.mockReset();+ generateTextMock.mockReset();});it('returns both title and description from a well-formed response', async () => {@@ -95,8 +102,91 @@ describe('generateGuideMeta', () => {expect(generateObjectMock).not.toHaveBeenCalled();});- it('returns null when the request throws', async () => {+ it('returns null when both the structured call and the text retry throw', async () => {generateObjectMock.mockRejectedValue(new Error('rate limited'));+ generateTextMock.mockRejectedValue(new Error('rate limited'));expect(await generateGuideMeta(steps, 'openai', 'gpt-4o-mini', 'key')).toBeNull();});});++describe('models without structured output support', () => {+ beforeEach(() => {+ generateObjectMock.mockReset();+ generateTextMock.mockReset();+ generateObjectMock.mockRejectedValue(UNSUPPORTED);+ });++ it('falls back to a plain text call and still yields a title', async () => {+ generateTextMock.mockResolvedValue({+ text: '{"title": "Check Recent Deaths on Wikipedia", "description": "Look up an entry."}',+ });++ const result = await generateGuideMeta(steps, 'openai', 'gpt-3.5-turbo', 'key');++ expect(result).toEqual({+ title: 'Check Recent Deaths on Wikipedia',+ description: 'Look up an entry.',+ });+ });++ it('asks the fallback call for bare JSON', async () => {+ generateTextMock.mockResolvedValue({ text: '{"title": "T"}' });++ await generateGuideMeta(steps, 'openai', 'gpt-3.5-turbo', 'key');++ const { prompt } = generateTextMock.mock.calls[0][0];+ expect(prompt).toContain('1. [https://admin.okta.com/users] Click Directory');+ expect(prompt).toContain('"title"');+ });++ it('does not spend a second call when the structured call succeeds', async () => {+ generateObjectMock.mockReset();+ generateObjectMock.mockResolvedValue({ object: { title: 'T', description: 'D' } });++ await generateGuideMeta(steps, 'openai', 'gpt-4o-mini', 'key');++ expect(generateTextMock).not.toHaveBeenCalled();+ });+});++describe('parseGuideMeta', () => {+ it('reads a bare JSON object', () => {+ expect(parseGuideMeta('{"title": "Okta Reset", "description": "Reset it."}')).toEqual({+ title: 'Okta Reset',+ description: 'Reset it.',+ });+ });++ it('unwraps a fenced code block the model added anyway', () => {+ expect(parseGuideMeta('```json\n{"title": "Okta Reset"}\n```')).toEqual({+ title: 'Okta Reset',+ description: undefined,+ });+ });++ it('ignores prose wrapped around the object', () => {+ expect(parseGuideMeta('Sure! Here you go:\n{"title": "Okta Reset"}\nHope that helps.')?.title).toBe('Okta Reset');+ });++ it('takes a bare line as the title when the model ignored the JSON request', () => {+ expect(parseGuideMeta('Reset a User Password in Okta')?.title).toBe('Reset a User Password in Okta');+ });++ it('refuses a bare line that is really a lead-in to broken JSON', () => {+ expect(parseGuideMeta('Here is the JSON: {"title": ')).toBeNull();+ });++ it('refuses a bare line too long to be a title', () => {+ expect(parseGuideMeta('x'.repeat(120))).toBeNull();+ });++ it('truncates an over-long title from the fallback path too', () => {+ const result = parseGuideMeta(JSON.stringify({ title: 'x'.repeat(90) }));+ expect(result?.title.length).toBe(70);+ expect(result?.title.endsWith('...')).toBe(true);+ });++ it('returns null for an empty response', () => {+ expect(parseGuideMeta(' ')).toBeNull();+ });+});diff --git a/src/core/capture/ai/__tests__/models.test.ts b/src/core/capture/ai/__tests__/models.test.tsnew file mode 100644index 0000000..d023619--- /dev/null+++ b/src/core/capture/ai/__tests__/models.test.ts@@ -0,0 +1,67 @@+import { readFileSync } from 'node:fs';+import { join } from 'node:path';+import { describe, expect, it } from 'vitest';+import { AI_PROVIDERS, CUSTOM_MODEL_VALUE, isCustomModel } from '../models';++const LOCALES = ['en', 'de', 'es', 'fr', 'pt-BR'];++const MODEL_KEYS = ['settings.model', 'settings.modelCustom'];++function localeKeys(locale: string): Set<string> {+ const keys = new Set<string>();+ let section = '';+ for (const line of readFileSync(join(process.cwd(), 'src/locales', `${locale}.yml`), 'utf8').split('\n')) {+ const top = /^([\w-]+):/.exec(line);+ if (top) {+ section = top[1];+ continue;+ }+ const nested = /^ {2}([\w-]+):/.exec(line);+ if (nested && section) keys.add(`${section}.${nested[1]}`);+ }+ return keys;+}++describe('isCustomModel', () => {+ it('treats a curated model as not custom', () => {+ expect(isCustomModel('gpt-4o-mini', AI_PROVIDERS.openai)).toBe(false);+ });++ it('reopens a stored model the provider list does not carry', () => {+ expect(isCustomModel('gpt-4o-2024-11-20', AI_PROVIDERS.openai)).toBe(true);+ });++ it('reads a model from another provider as custom', () => {+ expect(isCustomModel('claude-3-5-haiku-20241022', AI_PROVIDERS.openai)).toBe(true);+ });++ it('falls back to the curated list when nothing is stored', () => {+ expect(isCustomModel('', AI_PROVIDERS.openai)).toBe(false);+ expect(isCustomModel(' ', AI_PROVIDERS.openai)).toBe(false);+ });+});++describe('custom model sentinel', () => {+ it('never collides with a real model id', () => {+ for (const config of Object.values(AI_PROVIDERS)) {+ expect(config.models.some((option) => option.id === CUSTOM_MODEL_VALUE)).toBe(false);+ }+ });++ it('never renders as the empty value Radix rejects for an item', () => {+ expect(CUSTOM_MODEL_VALUE.length).toBeGreaterThan(0);+ });+});++describe('every provider default is selectable', () => {+ it.each(Object.entries(AI_PROVIDERS))('%s lists its own default model', (_key, config) => {+ expect(config.models.some((option) => option.id === config.defaultModel)).toBe(true);+ });+});++describe('model picker copy', () => {+ it.each(LOCALES)('%s carries every key the model picker renders', (locale) => {+ const keys = localeKeys(locale);+ for (const key of MODEL_KEYS) expect(keys).toContain(key);+ });+});diff --git a/src/core/capture/ai/meta.ts b/src/core/capture/ai/meta.tsindex fd95138..7f683a0 100644--- a/src/core/capture/ai/meta.ts+++ b/src/core/capture/ai/meta.ts@@ -1,7 +1,7 @@-import { generateObject, jsonSchema } from 'ai';+import { generateObject, generateText, jsonSchema } from 'ai';import { localStorage } from '@/lib/browser-api';import { logger } from '@/lib/logger';-import { GUIDE_META_PROMPT, getLanguageSuffix } from './prompts';+import { GUIDE_META_JSON_SUFFIX, GUIDE_META_PROMPT, getLanguageSuffix } from './prompts';import { createModel } from './provider';export interface GuideMeta {@@ -9,6 +9,10 @@ export interface GuideMeta {description?: string;}+const MAX_TITLE_LENGTH = 70;+const MAX_BARE_TITLE_LENGTH = 100;+const FENCE = /^```(?:json)?\s*|\s*```$/g;+const guideMetaSchema = jsonSchema<{ title: string; description?: string | null }>({type: 'object',properties: {@@ -19,6 +23,36 @@ const guideMetaSchema = jsonSchema<{ title: string; description?: string | nulladditionalProperties: false,});+function toGuideMeta(rawTitle: unknown, rawDescription: unknown): GuideMeta | null {+ let title = typeof rawTitle === 'string' ? rawTitle.trim().replace(/^"|"$/g, '') : '';+ if (!title) return null;+ if (title.length > MAX_TITLE_LENGTH) title = `${title.slice(0, MAX_TITLE_LENGTH - 3)}...`;++ const description =+ typeof rawDescription === 'string' ? rawDescription.trim().replace(/^"|"$/g, '') || undefined : undefined;+ return { title, description };+}++export function parseGuideMeta(raw: string): GuideMeta | null {+ const text = raw.trim().replace(FENCE, '').trim();++ const start = text.indexOf('{');+ const end = text.lastIndexOf('}');+ if (start !== -1 && end > start) {+ try {+ const parsed = JSON.parse(text.slice(start, end + 1)) as Record<string, unknown>;+ const meta = toGuideMeta(parsed.title, parsed.description);+ if (meta) return meta;+ } catch {+ logger.warn('Guide meta fallback returned malformed JSON');+ }+ }++ const line = text.split('\n')[0]?.trim() ?? '';+ if (!line || line.length > MAX_BARE_TITLE_LENGTH || line.includes('{')) return null;+ return toGuideMeta(line, null);+}+export async function generateGuideMeta(steps: { description: string; url: string }[],provider: string,@@ -28,23 +62,30 @@ export async function generateGuideMeta(if (steps.length === 0) return null;const formatted = steps.map((s, i) => `${i + 1}. [${s.url}] ${s.description}`).join('\n');+ const settings = await localStorage.get(['aiLanguage']);+ const locale = (settings.aiLanguage as string) || 'en';+ const prompt = GUIDE_META_PROMPT.replace('{{steps}}', formatted) + getLanguageSuffix(locale);+ const aiModel = createModel(provider, model, apiKey);try {- const settings = await localStorage.get(['aiLanguage']);- const locale = (settings.aiLanguage as string) || 'en';const { object } = await generateObject({- model: createModel(provider, model, apiKey),+ model: aiModel,schema: guideMetaSchema,- prompt: GUIDE_META_PROMPT.replace('{{steps}}', formatted) + getLanguageSuffix(locale),+ prompt,maxOutputTokens: 200,});+ return toGuideMeta(object.title, object.description);+ } catch (err) {+ logger.warn('Structured guide meta failed, retrying as plain text', err);+ }- let title = object.title?.trim().replace(/^"|"$/g, '') ?? '';- if (!title) return null;- if (title.length > 70) title = `${title.slice(0, 67)}...`;-- const description = object.description?.trim().replace(/^"|"$/g, '') || undefined;- return { title, description };+ try {+ const { text } = await generateText({+ model: aiModel,+ prompt: prompt + GUIDE_META_JSON_SUFFIX,+ maxOutputTokens: 200,+ });+ return parseGuideMeta(text);} catch (err) {logger.error('Guide meta generation failed', err);return null;diff --git a/src/core/capture/ai/models.ts b/src/core/capture/ai/models.tsindex 9058efb..9c7685a 100644--- a/src/core/capture/ai/models.ts+++ b/src/core/capture/ai/models.ts@@ -32,3 +32,10 @@ export const AI_PROVIDERS: Record<string, AIProviderConfig> = {};export type AIProviderKey = keyof typeof AI_PROVIDERS;++export const CUSTOM_MODEL_VALUE = 'mimik-custom-model';++export function isCustomModel(model: string, provider: AIProviderConfig): boolean {+ const id = model.trim();+ return id.length > 0 && !provider.models.some((option) => option.id === id);+}diff --git a/src/core/capture/ai/prompts.ts b/src/core/capture/ai/prompts.tsindex 47e4ebd..12077e4 100644--- a/src/core/capture/ai/prompts.ts+++ b/src/core/capture/ai/prompts.ts@@ -30,6 +30,10 @@ Examples of good descriptions:- "Reset a locked-out user's password from the Okta admin panel. For IT support staff."- "Configure which Slack channels send desktop notifications, and set a do-not-disturb schedule."`;+export const GUIDE_META_JSON_SUFFIX = `++Reply with nothing but a JSON object shaped {"title": string, "description": string}. No code fence, no commentary.`;+export const REWRITE_PROMPT = `You are editing one span of text inside a browser workflow guide. The text describes a step a reader must perform, or summarises what the workflow accomplishes.Selected text:diff --git a/src/locales/de.yml b/src/locales/de.ymlindex 83a4402..5f84c21 100644--- a/src/locales/de.yml+++ b/src/locales/de.yml@@ -96,6 +96,7 @@ settings:targetColor: "Markenfarbe"title: EinstellungenaiDescriptions: KI-Beschreibungen+ modelCustom: Eigenes Modellprovider: Anbietermodel: ModellapiKey: API-Schlüsseldiff --git a/src/locales/en.yml b/src/locales/en.ymlindex 287f43d..f38167f 100644--- a/src/locales/en.yml+++ b/src/locales/en.yml@@ -84,6 +84,7 @@ settings:aiDescriptions: AI Descriptionsprovider: Providermodel: Model+ modelCustom: Custom modelapiKey: API KeyaiLanguage: Description LanguagevoiceNarration: Voice Narrationdiff --git a/src/locales/es.yml b/src/locales/es.ymlindex 154bccb..bbfaa04 100644--- a/src/locales/es.yml+++ b/src/locales/es.yml@@ -84,6 +84,7 @@ settings:aiDescriptions: Descripciones con IAprovider: Proveedormodel: Modelo+ modelCustom: Modelo personalizadoapiKey: Clave APIaiLanguage: Idioma de descripcionesvoiceNarration: "Narración por voz"diff --git a/src/locales/fr.yml b/src/locales/fr.ymlindex 88232bf..678b20b 100644--- a/src/locales/fr.yml+++ b/src/locales/fr.yml@@ -84,6 +84,7 @@ settings:aiDescriptions: Descriptions IAprovider: Fournisseurmodel: "Modèle"+ modelCustom: Modèle personnaliséapiKey: "Clé API"aiLanguage: Langue des descriptionsvoiceNarration: "Narration vocale"diff --git a/src/locales/pt-BR.yml b/src/locales/pt-BR.ymlindex 9c2b78f..7a204b6 100644--- a/src/locales/pt-BR.yml+++ b/src/locales/pt-BR.yml@@ -84,6 +84,7 @@ settings:aiDescriptions: Descrições com IAprovider: Provedormodel: Modelo+ modelCustom: Modelo personalizadoapiKey: Chave da APIaiLanguage: "Idioma das descrições"voiceNarration: "Narração por voz"