Skip to content

Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.

feat: Mercury sees photos and PDFs, and listens to voice notes, from chat

Salman Qureshidacf04606b8e
diff --git a/src/core/agent.ts b/src/core/agent.ts
index 564d3cc4..af95de9f 100644
--- a/src/core/agent.ts
+++ b/src/core/agent.ts
@@ -96,6 +96,7 @@ import { createTaskSurface } from './task-surface.js';
import { buildFileChangePreview } from '../utils/file-preview.js';
import { whatsNewText } from '../utils/whats-new.js';
import { handleBotsCommand as handleBotsCommandImpl } from './commands/bots-command.js';
+import { withAttachments } from './attachments.js';
import { handlePluginsCommand, tryPluginCommand, pluginCommandHelp, hasPluginDialog, answerPluginDialog } from '../plugins/commands.js';
import { handleChatCommand as handleChatCommandImpl } from './commands/chat-command.js';
import { handleBgCommand as handleBgCommandImpl } from './commands/bg-command.js';
@@ -3148,7 +3149,8 @@ export class Agent {
// DeepSeek thinking models require prior reasoning_content to be
// passed back with assistant turns (issue #24); other providers
// get plain text.
- const providerMessages = deepseekProviderOptions ? withReasoningParts(messages) : messages;
+ // Photos and PDFs from the chat go to models that can take them.
+ const providerMessages = withAttachments(deepseekProviderOptions ? withReasoningParts(messages) : messages, msg, provider.name, provider.getModel());
// Per-provider output ceiling: a 32K request is rejected outright by
// providers with smaller limits and burned a halve-and-retry round.
const providerMaxOutputTokens = Math.min(effectiveMaxOutputTokens, provider.getMaxOutputTokens() ?? Infinity);
diff --git a/src/core/attachments.test.ts b/src/core/attachments.test.ts
new file mode 100644
index 00000000..5ce53a20
--- /dev/null
+++ b/src/core/attachments.test.ts
@@ -0,0 +1,58 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, describe, expect, it } from 'vitest';
+import { modelReadsPdf, modelSeesImages, withAttachments } from './attachments.js';
+import type { ChannelMessage } from '../types/channel.js';
+
+const dir = mkdtempSync(join(tmpdir(), 'mercury-att-'));
+mkdirSync(dir, { recursive: true });
+const img = join(dir, 'image-1.jpg');
+const pdf = join(dir, 'document-2.pdf');
+writeFileSync(img, Buffer.from([0xff, 0xd8, 0xff, 0xe0, 1, 2, 3]));
+writeFileSync(pdf, '%PDF-1.4 test');
+afterAll(() => rmSync(dir, { recursive: true, force: true }));
+
+const msg = (attachments: unknown[]): ChannelMessage => ({
+ id: '1', channelId: 'whatsapp:x', channelType: 'whatsapp', senderId: 'o', content: 'What is it about?', timestamp: 0,
+ metadata: { attachments },
+});
+const history = [{ role: 'user', content: 'earlier' }, { role: 'assistant', content: 'ok' }, { role: 'user', content: 'What is it about?' }];
+
+describe('chat attachments for the model', () => {
+ it('a photo goes to a vision model as an image part on the current message only', () => {
+ const out = withAttachments(history, msg([{ kind: 'image', path: img }]), 'openai', 'gpt-4.1-mini');
+ expect(out[0]).toBe(history[0]);
+ const content = out[2].content as unknown as Array<Record<string, unknown>>;
+ expect(content[0]).toEqual({ type: 'text', text: 'What is it about?' });
+ expect(content[1]).toMatchObject({ type: 'image', mediaType: 'image/jpeg' });
+ expect(Buffer.isBuffer(content[1].image)).toBe(true);
+ });
+
+ it('a text-only model gets a note instead of an image it would reject', () => {
+ const out = withAttachments(history, msg([{ kind: 'image', path: img }]), 'deepseek', 'deepseek-chat');
+ const content = out[2].content as unknown as Array<Record<string, unknown>>;
+ expect(content).toHaveLength(1);
+ expect(String(content[0].text)).toMatch(/can't view images/);
+ });
+
+ it('PDFs go to models that read them, and are left as paths otherwise', () => {
+ const claude = withAttachments(history, msg([{ kind: 'document', path: pdf }]), 'anthropic', 'claude-sonnet-4-5');
+ expect((claude[2].content as unknown as Array<Record<string, unknown>>)[1]).toMatchObject({ type: 'file', mediaType: 'application/pdf', filename: 'document-2.pdf' });
+ expect(withAttachments(history, msg([{ kind: 'document', path: pdf }]), 'ollama', 'qwen3:8b')).toBe(history);
+ });
+
+ it('no attachments, missing files: unchanged', () => {
+ expect(withAttachments(history, msg([]), 'openai', 'gpt-4o')).toBe(history);
+ expect(withAttachments(history, msg([{ kind: 'image', path: join(dir, 'gone.jpg') }]), 'openai', 'gpt-4o')).toBe(history);
+ });
+
+ it('model capability checks', () => {
+ expect(modelSeesImages('openai', 'gpt-4.1-mini')).toBe(true);
+ expect(modelSeesImages('anthropic', 'claude-opus-4-1')).toBe(true);
+ expect(modelSeesImages('deepseek', 'deepseek-chat')).toBe(false);
+ expect(modelSeesImages('ollama', 'llama3.1:8b')).toBe(false);
+ expect(modelReadsPdf('openai', 'gpt-5.2-chat-latest')).toBe(true);
+ expect(modelReadsPdf('openai', 'gpt-3.5-turbo')).toBe(false);
+ });
+});
diff --git a/src/core/attachments.ts b/src/core/attachments.ts
new file mode 100644
index 00000000..db21d7d9
--- /dev/null
+++ b/src/core/attachments.ts
@@ -0,0 +1,91 @@
+/**
+ * Files a chat message carries (photos, PDFs) handed to the model as real
+ * content, so "what is this?" about a photo works instead of the model seeing
+ * only a file path. Channels put them in `msg.metadata.attachments` as
+ * `{ kind, path, mime?, name? }` (WhatsApp does today).
+ *
+ * Only models that can take them get them: a text-only model would reject
+ * the whole request, so it gets the path and a note instead.
+ */
+import { existsSync, readFileSync, statSync } from 'node:fs';
+import { basename, extname } from 'node:path';
+import type { ChannelMessage } from '../types/channel.js';
+
+const MAX_IMAGE_BYTES = 8 * 1024 * 1024;
+const MAX_PDF_BYTES = 15 * 1024 * 1024;
+const MAX_FILES = 4;
+
+interface Attachment {
+ kind: string;
+ path: string;
+ mime?: string;
+ name?: string;
+}
+
+const IMAGE_MIME: Record<string, string> = { '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.png': 'image/png', '.gif': 'image/gif', '.webp': 'image/webp' };
+
+/** Models that accept image input. Conservative: unknown models get the note instead. */
+export function modelSeesImages(provider: string, model: string): boolean {
+ const m = model.toLowerCase();
+ if (provider === 'deepseek') return false;
+ if (/gpt-4o|gpt-4\.1|gpt-5|chatgpt-4o|\bo[134](-|$)|claude|gemini|pixtral|llava|qwen.*vl|vision|grok-(2-vision|4)|llama-?4|mistral-(medium|small)-3|gemma-?3/.test(m)) return true;
+ return false;
+}
+
+/** Models that accept PDF documents directly. */
+export function modelReadsPdf(provider: string, model: string): boolean {
+ const m = model.toLowerCase();
+ if (provider === 'anthropic' || /claude/.test(m)) return true;
+ if (/gemini/.test(m)) return true;
+ if (provider === 'openai' && /gpt-4o|gpt-4\.1|gpt-5|\bo[34](-|$)/.test(m)) return true;
+ return false;
+}
+
+export function attachmentsOf(msg: ChannelMessage): Attachment[] {
+ const raw = msg.metadata?.attachments;
+ if (!Array.isArray(raw)) return [];
+ return raw
+ .filter((a): a is Attachment => !!a && typeof (a as Attachment).path === 'string' && typeof (a as Attachment).kind === 'string')
+ .slice(0, MAX_FILES);
+}
+
+function isPdf(a: Attachment): boolean {
+ return a.mime === 'application/pdf' || extname(a.path).toLowerCase() === '.pdf';
+}
+
+/**
+ * The current turn's user message with its photos and PDFs attached, for
+ * this provider and model. Returns the messages unchanged when there is
+ * nothing to attach or the model can't take it.
+ */
+export function withAttachments<T extends { role: string; content: unknown }>(messages: T[], msg: ChannelMessage, provider: string, model: string): T[] {
+ const files = attachmentsOf(msg);
+ if (!files.length) return messages;
+ const parts: Array<Record<string, unknown>> = [];
+ const notes: string[] = [];
+ for (const a of files) {
+ if (!existsSync(a.path)) continue;
+ const size = statSync(a.path).size;
+ if (a.kind === 'image') {
+ const mediaType = a.mime?.startsWith('image/') ? a.mime : IMAGE_MIME[extname(a.path).toLowerCase()] ?? 'image/jpeg';
+ if (!modelSeesImages(provider, model)) notes.push(`The current model (${model}) can't view images; say so, and offer to describe it with a vision model if one is configured.`);
+ else if (size > MAX_IMAGE_BYTES) notes.push(`An image is too large to view (${Math.round(size / 1048576)} MB).`);
+ else parts.push({ type: 'image', image: readFileSync(a.path), mediaType });
+ } else if (isPdf(a)) {
+ if (modelReadsPdf(provider, model) && size <= MAX_PDF_BYTES) parts.push({ type: 'file', data: readFileSync(a.path), mediaType: 'application/pdf', filename: a.name ?? basename(a.path) });
+ }
+ }
+ if (!parts.length && !notes.length) return messages;
+ // The last user message is this turn's (history comes first).
+ let idx = -1;
+ for (let i = messages.length - 1; i >= 0; i--) {
+ if (messages[i].role === 'user') { idx = i; break; }
+ }
+ if (idx < 0) return messages;
+ const current = messages[idx];
+ const text = typeof current.content === 'string' ? current.content : msg.content;
+ const content = [{ type: 'text', text: notes.length ? `${text}\n\n(${notes.join(' ')})` : text }, ...parts];
+ const out = messages.slice();
+ out[idx] = { ...current, content } as T;
+ return out;
+}
diff --git a/src/plugins/channel-bridge.ts b/src/plugins/channel-bridge.ts
index 4c8713a5..928e58d7 100644
--- a/src/plugins/channel-bridge.ts
+++ b/src/plugins/channel-bridge.ts
@@ -170,6 +170,8 @@ export class PluginChannel extends BaseChannel {
private readonly notifyOwner: (message: string) => void,
/** Persist a plugin setting without restarting it (access level, members). */
private readonly saveSetting: (key: string, value: unknown) => Promise<void> = async () => {},
+ /** Speech to text for voice notes; null when no provider can do it. */
+ private readonly transcribe: ((path: string) => Promise<string>) | null = null,
) {
super();
this.type = manifest.contributes.channel!.type as ChannelType;
@@ -389,16 +391,55 @@ export class PluginChannel extends BaseChannel {
.slice(0, 10)
.map((a) => ({ path: a.path as string, kind: String(a.kind ?? 'document'), name: typeof a.name === 'string' ? a.name : basename(a.path as string), mime: typeof a.mime === 'string' ? a.mime : undefined }))
: [];
- let content = text;
- if (files.length) {
- const list = files.map((f) => `${f.kind}: ${f.path}`).join('\n');
- content = `${content}${content ? '\n' : ''}[Attachments]\n${list}`;
- }
- if (!content) return;
+ // Voice notes become text first (in arrival order per chat), then go to the agent.
+ const voice = files.filter((f) => f.kind === 'voice' || f.kind === 'audio');
+ const others = files.filter((f) => !voice.includes(f));
+ const chatId = p.chatId;
+ const prev = this.chatQueue.get(chatId) ?? Promise.resolve();
+ const next = prev.then(async () => {
+ let content = text;
+ let voiceNote = false;
+ for (const v of voice) {
+ if (!this.transcribe) {
+ await this.send('🎤 I can\'t listen to voice notes yet: that needs an OpenAI provider. Add your OpenAI key once in the dashboard (Providers) or with `mercury doctor`. Meanwhile, please type it.', `${this.type}:${chatId}`).catch(() => {});
+ continue;
+ }
+ try {
+ const said = await this.transcribe(v.path);
+ if (said) {
+ content = `${content}${content ? '\n' : ''}${said}`;
+ voiceNote = true;
+ }
+ } catch (err) {
+ await this.send(`🎤 I couldn't understand that voice note (${(err as Error).message.slice(0, 160)}). Could you type it?`, `${this.type}:${chatId}`).catch(() => {});
+ }
+ }
+ if (others.length) {
+ const list = others.map((f) => `${f.kind}: ${f.path}`).join('\n');
+ content = `${content}${content ? '\n' : ''}[Attachments: the files are attached to this message; these are their paths on this computer]\n${list}`;
+ }
+ if (!content) return;
+ this.emitMessage(id, p, sender, role, content, others, voiceNote);
+ }).catch(() => {});
+ this.chatQueue.set(chatId, next);
+ void next.finally(() => { if (this.chatQueue.get(chatId) === next) this.chatQueue.delete(chatId); });
+ }
+
+ private chatQueue = new Map<string, Promise<void>>();
+
+ private emitMessage(
+ id: string,
+ p: IncomingParams,
+ sender: string,
+ role: 'admin' | 'member',
+ content: string,
+ files: Array<{ path: string; kind: string; name: string; mime?: string }>,
+ voiceNote: boolean,
+ ): void {
const msg: ChannelMessage = {
id,
- channelId: `${this.type}:${p.chatId}`,
+ channelId: `${this.type}:${p.chatId as string}`,
channelType: this.type,
senderId: sender,
senderName: typeof p.senderName === 'string' ? p.senderName.slice(0, 100) : undefined,
@@ -410,8 +451,9 @@ export class PluginChannel extends BaseChannel {
chatId: p.chatId,
isGroup: p.isGroup === true,
attachments: files,
+ voiceNote,
channelAccess: this.accessFor(role),
- channelGuidance: this.guidance(role, p.isGroup === true),
+ channelGuidance: this.guidance(role, p.isGroup === true) + (voiceNote ? '\nThey sent this as a voice note (transcribed above); reply in text as usual.' : ''),
},
};
this.emit(msg);
diff --git a/src/plugins/channel.test.ts b/src/plugins/channel.test.ts
index 13c621ea..1b8c6cfa 100644
--- a/src/plugins/channel.test.ts
+++ b/src/plugins/channel.test.ts
@@ -5,8 +5,9 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { PermissionManager } from '../capabilities/permissions.js';
import type { Channel } from '../channels/base.js';
import type { ChannelMessage } from '../types/channel.js';
-import { PluginManager } from './manager.js';
-import { getPluginChannel, setChannelHost, splitMessage, toWhatsAppText, type PluginChannel } from './channel-bridge.js';
+import { PluginManager, setProviderLookup } from './manager.js';
+import { PluginChannel } from './channel-bridge.js';
+import { getPluginChannel, setChannelHost, splitMessage, toWhatsAppText } from './channel-bridge.js';
import { getPluginTools, resetToolBridge } from './tool-bridge.js';
import { pluginDataDir } from './store.js';
import { manifestSchema } from './types.js';
@@ -105,6 +106,7 @@ beforeEach(() => {
register: (type, channel) => { registered.push({ type, channel }); channel.onMessage((m) => received.push(m)); },
unregister: (type, channel) => { registered = registered.filter((r) => r.channel !== channel); },
});
+ setProviderLookup(() => undefined); // never the real ~/.mercury providers
mgr = new PluginManager({ safeMode: false, notify: (m) => { notices.push(m); } });
});
@@ -333,4 +335,48 @@ describe('plugin channels (plugin API 1.1)', () => {
expect(received[2].metadata?.channelAccess).toBe('general');
});
});
+
+ describe('voice notes', () => {
+ function channelWith(transcribe: ((p: string) => Promise<string>) | null) {
+ const sent: string[] = [];
+ const got: ChannelMessage[] = [];
+ const manifest = manifestSchema.parse(JSON.parse(readFileSync(join(channelPlugin(src, 'voice-notes'), 'mercury-plugin.json'), 'utf8')));
+ const ch = new PluginChannel(
+ manifest,
+ { request: async (method, params) => { if (method === 'mercury/channel/send') sent.push((params as { text: string }).text); return {}; } },
+ () => ({ owner: '+15550002', access: 'extended' }),
+ () => {},
+ async () => {},
+ transcribe,
+ );
+ ch.onMessage((m) => got.push(m));
+ return { ch, sent, got };
+ }
+ const note = (id: string, text?: string) => ({ id, chatId: '[email protected]', senderId: 'x@lid', senderPhone: '15550002', text, attachments: [{ kind: 'voice', path: `/tmp/${id}.ogg` }] });
+
+ it('are transcribed and reach the agent as text, in order', async () => {
+ const { ch, got } = channelWith(async (p) => { if (p.includes('v1')) await sleep(60); return p.includes('v1') ? 'What is the news today?' : 'And the weather?'; });
+ ch.handleNotification('mercury/channel/message', note('v1'));
+ ch.handleNotification('mercury/channel/message', note('v2'));
+ await waitFor(() => got.length === 2);
+ expect(got.map((m) => m.content)).toEqual(['What is the news today?', 'And the weather?']);
+ expect(got[0].metadata?.voiceNote).toBe(true);
+ expect(String(got[0].metadata?.channelGuidance)).toMatch(/voice note/);
+ });
+
+ it('without a transcriber: the owner is told how to enable it; nothing reaches the agent', async () => {
+ const { ch, sent, got } = channelWith(null);
+ ch.handleNotification('mercury/channel/message', note('v3'));
+ await waitFor(() => sent.length === 1);
+ expect(sent[0]).toMatch(/OpenAI provider[\s\S]*Providers/);
+ expect(got).toEqual([]);
+ });
+
+ it('a failed transcription asks the owner to type it', async () => {
+ const { ch, sent } = channelWith(async () => { throw new Error('insufficient_quota'); });
+ ch.handleNotification('mercury/channel/message', note('v4'));
+ await waitFor(() => sent.length === 1);
+ expect(sent[0]).toMatch(/couldn't understand that voice note \(insufficient_quota\)/);
+ });
+ });
});
diff --git a/src/plugins/manager.ts b/src/plugins/manager.ts
index 27b9829c..4dbaba3d 100644
--- a/src/plugins/manager.ts
+++ b/src/plugins/manager.ts
@@ -33,6 +33,7 @@ import { setToolOutcomeSink, unpublishPlugin, listLiveTools, listPluginCommands
import { InProcessRuntime, inProcessLogger } from './runtime/in-process.js';
import { OutOfProcessRuntime } from './runtime/out-of-process.js';
import { PluginChannel, type ChannelStatus } from './channel-bridge.js';
+import { transcribeAudio } from './transcribe.js';
import type { PluginRuntime } from './runtime/types.js';
import { providerSecretNames, type CatalogEntry, type DisabledCode, type PluginListItem, type PluginManifest, type PluginProviderId, type PluginsState } from './types.js';
import { loadConfig } from '../utils/config.js';
@@ -275,6 +276,11 @@ export class PluginManager {
() => resolvedSettings(manifest),
(m) => { void this.notifyFn(m); },
(key, value) => this.setSetting(id, key, value),
+ // Voice notes: transcribed with the owner's OpenAI provider, when there is one.
+ (() => {
+ const openai = providerLookup('openai');
+ return openai?.apiKey ? (path: string) => transcribeAudio(path, { apiKey: openai.apiKey!, baseUrl: openai.baseUrl }) : null;
+ })(),
);
}
const deps = {
diff --git a/src/plugins/transcribe.ts b/src/plugins/transcribe.ts
new file mode 100644
index 00000000..1eac01cb
--- /dev/null
+++ b/src/plugins/transcribe.ts
@@ -0,0 +1,37 @@
+/**
+ * Speech to text for voice notes, with the OpenAI provider the owner already
+ * set up in Mercury (gpt-4o-mini-transcribe, falling back to whisper-1).
+ */
+import { readFileSync, statSync } from 'node:fs';
+import { basename } from 'node:path';
+
+const MAX_BYTES = 25 * 1024 * 1024;
+
+export interface TranscribeCreds {
+ apiKey: string;
+ baseUrl?: string;
+}
+
+export async function transcribeAudio(path: string, creds: TranscribeCreds, signal?: AbortSignal): Promise<string> {
+ if (statSync(path).size > MAX_BYTES) throw new Error('the voice note is longer than the 25 MB transcription limit');
+ const base = (creds.baseUrl || 'https://api.openai.com/v1').replace(/\/+$/, '');
+ const bytes = readFileSync(path);
+ let lastError = '';
+ for (const model of ['gpt-4o-mini-transcribe', 'whisper-1']) {
+ const form = new FormData();
+ form.append('model', model);
+ form.append('file', new Blob([bytes], { type: 'audio/ogg' }), basename(path).replace(/\.[^.]+$/, '') + '.ogg');
+ const res = await fetch(`${base}/audio/transcriptions`, {
+ method: 'POST',
+ headers: { Authorization: `Bearer ${creds.apiKey}` },
+ body: form,
+ signal: signal ?? AbortSignal.timeout(60_000),
+ });
+ const body = (await res.json().catch(() => ({}))) as { text?: string; error?: { message?: string; code?: string } };
+ if (res.ok && typeof body.text === 'string') return body.text.trim();
+ lastError = body.error?.message ?? `HTTP ${res.status}`;
+ // Only a missing model is worth retrying with the other one.
+ if (!/model/i.test(lastError) || res.status === 401 || res.status === 429) break;
+ }
+ throw new Error(lastError);
+}