Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.
feat: Mercury sees photos and PDFs, and listens to voice notes, from chat
Salman Qureshidacf04606b8e
diff --git a/src/core/agent.ts b/src/core/agent.tsindex 564d3cc4..af95de9f 100644--- a/src/core/agent.ts+++ b/src/core/agent.ts@@ -96,6 +96,7 @@ import { createTaskSurface } from './task-surface.js';import { buildFileChangePreview } from '../utils/file-preview.js';import { whatsNewText } from '../utils/whats-new.js';import { handleBotsCommand as handleBotsCommandImpl } from './commands/bots-command.js';+import { withAttachments } from './attachments.js';import { handlePluginsCommand, tryPluginCommand, pluginCommandHelp, hasPluginDialog, answerPluginDialog } from '../plugins/commands.js';import { handleChatCommand as handleChatCommandImpl } from './commands/chat-command.js';import { handleBgCommand as handleBgCommandImpl } from './commands/bg-command.js';@@ -3148,7 +3149,8 @@ export class Agent {// DeepSeek thinking models require prior reasoning_content to be// passed back with assistant turns (issue #24); other providers// get plain text.- const providerMessages = deepseekProviderOptions ? withReasoningParts(messages) : messages;+ // Photos and PDFs from the chat go to models that can take them.+ const providerMessages = withAttachments(deepseekProviderOptions ? withReasoningParts(messages) : messages, msg, provider.name, provider.getModel());// Per-provider output ceiling: a 32K request is rejected outright by// providers with smaller limits and burned a halve-and-retry round.const providerMaxOutputTokens = Math.min(effectiveMaxOutputTokens, provider.getMaxOutputTokens() ?? Infinity);diff --git a/src/core/attachments.test.ts b/src/core/attachments.test.tsnew file mode 100644index 00000000..5ce53a20--- /dev/null+++ b/src/core/attachments.test.ts@@ -0,0 +1,58 @@+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterAll, describe, expect, it } from 'vitest';+import { modelReadsPdf, modelSeesImages, withAttachments } from './attachments.js';+import type { ChannelMessage } from '../types/channel.js';++const dir = mkdtempSync(join(tmpdir(), 'mercury-att-'));+mkdirSync(dir, { recursive: true });+const img = join(dir, 'image-1.jpg');+const pdf = join(dir, 'document-2.pdf');+writeFileSync(img, Buffer.from([0xff, 0xd8, 0xff, 0xe0, 1, 2, 3]));+writeFileSync(pdf, '%PDF-1.4 test');+afterAll(() => rmSync(dir, { recursive: true, force: true }));++const msg = (attachments: unknown[]): ChannelMessage => ({+ id: '1', channelId: 'whatsapp:x', channelType: 'whatsapp', senderId: 'o', content: 'What is it about?', timestamp: 0,+ metadata: { attachments },+});+const history = [{ role: 'user', content: 'earlier' }, { role: 'assistant', content: 'ok' }, { role: 'user', content: 'What is it about?' }];++describe('chat attachments for the model', () => {+ it('a photo goes to a vision model as an image part on the current message only', () => {+ const out = withAttachments(history, msg([{ kind: 'image', path: img }]), 'openai', 'gpt-4.1-mini');+ expect(out[0]).toBe(history[0]);+ const content = out[2].content as unknown as Array<Record<string, unknown>>;+ expect(content[0]).toEqual({ type: 'text', text: 'What is it about?' });+ expect(content[1]).toMatchObject({ type: 'image', mediaType: 'image/jpeg' });+ expect(Buffer.isBuffer(content[1].image)).toBe(true);+ });++ it('a text-only model gets a note instead of an image it would reject', () => {+ const out = withAttachments(history, msg([{ kind: 'image', path: img }]), 'deepseek', 'deepseek-chat');+ const content = out[2].content as unknown as Array<Record<string, unknown>>;+ expect(content).toHaveLength(1);+ expect(String(content[0].text)).toMatch(/can't view images/);+ });++ it('PDFs go to models that read them, and are left as paths otherwise', () => {+ const claude = withAttachments(history, msg([{ kind: 'document', path: pdf }]), 'anthropic', 'claude-sonnet-4-5');+ expect((claude[2].content as unknown as Array<Record<string, unknown>>)[1]).toMatchObject({ type: 'file', mediaType: 'application/pdf', filename: 'document-2.pdf' });+ expect(withAttachments(history, msg([{ kind: 'document', path: pdf }]), 'ollama', 'qwen3:8b')).toBe(history);+ });++ it('no attachments, missing files: unchanged', () => {+ expect(withAttachments(history, msg([]), 'openai', 'gpt-4o')).toBe(history);+ expect(withAttachments(history, msg([{ kind: 'image', path: join(dir, 'gone.jpg') }]), 'openai', 'gpt-4o')).toBe(history);+ });++ it('model capability checks', () => {+ expect(modelSeesImages('openai', 'gpt-4.1-mini')).toBe(true);+ expect(modelSeesImages('anthropic', 'claude-opus-4-1')).toBe(true);+ expect(modelSeesImages('deepseek', 'deepseek-chat')).toBe(false);+ expect(modelSeesImages('ollama', 'llama3.1:8b')).toBe(false);+ expect(modelReadsPdf('openai', 'gpt-5.2-chat-latest')).toBe(true);+ expect(modelReadsPdf('openai', 'gpt-3.5-turbo')).toBe(false);+ });+});diff --git a/src/core/attachments.ts b/src/core/attachments.tsnew file mode 100644index 00000000..db21d7d9--- /dev/null+++ b/src/core/attachments.ts@@ -0,0 +1,91 @@+/**+ * Files a chat message carries (photos, PDFs) handed to the model as real+ * content, so "what is this?" about a photo works instead of the model seeing+ * only a file path. Channels put them in `msg.metadata.attachments` as+ * `{ kind, path, mime?, name? }` (WhatsApp does today).+ *+ * Only models that can take them get them: a text-only model would reject+ * the whole request, so it gets the path and a note instead.+ */+import { existsSync, readFileSync, statSync } from 'node:fs';+import { basename, extname } from 'node:path';+import type { ChannelMessage } from '../types/channel.js';++const MAX_IMAGE_BYTES = 8 * 1024 * 1024;+const MAX_PDF_BYTES = 15 * 1024 * 1024;+const MAX_FILES = 4;++interface Attachment {+ kind: string;+ path: string;+ mime?: string;+ name?: string;+}++const IMAGE_MIME: Record<string, string> = { '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.png': 'image/png', '.gif': 'image/gif', '.webp': 'image/webp' };++/** Models that accept image input. Conservative: unknown models get the note instead. */+export function modelSeesImages(provider: string, model: string): boolean {+ const m = model.toLowerCase();+ if (provider === 'deepseek') return false;+ if (/gpt-4o|gpt-4\.1|gpt-5|chatgpt-4o|\bo[134](-|$)|claude|gemini|pixtral|llava|qwen.*vl|vision|grok-(2-vision|4)|llama-?4|mistral-(medium|small)-3|gemma-?3/.test(m)) return true;+ return false;+}++/** Models that accept PDF documents directly. */+export function modelReadsPdf(provider: string, model: string): boolean {+ const m = model.toLowerCase();+ if (provider === 'anthropic' || /claude/.test(m)) return true;+ if (/gemini/.test(m)) return true;+ if (provider === 'openai' && /gpt-4o|gpt-4\.1|gpt-5|\bo[34](-|$)/.test(m)) return true;+ return false;+}++export function attachmentsOf(msg: ChannelMessage): Attachment[] {+ const raw = msg.metadata?.attachments;+ if (!Array.isArray(raw)) return [];+ return raw+ .filter((a): a is Attachment => !!a && typeof (a as Attachment).path === 'string' && typeof (a as Attachment).kind === 'string')+ .slice(0, MAX_FILES);+}++function isPdf(a: Attachment): boolean {+ return a.mime === 'application/pdf' || extname(a.path).toLowerCase() === '.pdf';+}++/**+ * The current turn's user message with its photos and PDFs attached, for+ * this provider and model. Returns the messages unchanged when there is+ * nothing to attach or the model can't take it.+ */+export function withAttachments<T extends { role: string; content: unknown }>(messages: T[], msg: ChannelMessage, provider: string, model: string): T[] {+ const files = attachmentsOf(msg);+ if (!files.length) return messages;+ const parts: Array<Record<string, unknown>> = [];+ const notes: string[] = [];+ for (const a of files) {+ if (!existsSync(a.path)) continue;+ const size = statSync(a.path).size;+ if (a.kind === 'image') {+ const mediaType = a.mime?.startsWith('image/') ? a.mime : IMAGE_MIME[extname(a.path).toLowerCase()] ?? 'image/jpeg';+ if (!modelSeesImages(provider, model)) notes.push(`The current model (${model}) can't view images; say so, and offer to describe it with a vision model if one is configured.`);+ else if (size > MAX_IMAGE_BYTES) notes.push(`An image is too large to view (${Math.round(size / 1048576)} MB).`);+ else parts.push({ type: 'image', image: readFileSync(a.path), mediaType });+ } else if (isPdf(a)) {+ if (modelReadsPdf(provider, model) && size <= MAX_PDF_BYTES) parts.push({ type: 'file', data: readFileSync(a.path), mediaType: 'application/pdf', filename: a.name ?? basename(a.path) });+ }+ }+ if (!parts.length && !notes.length) return messages;+ // The last user message is this turn's (history comes first).+ let idx = -1;+ for (let i = messages.length - 1; i >= 0; i--) {+ if (messages[i].role === 'user') { idx = i; break; }+ }+ if (idx < 0) return messages;+ const current = messages[idx];+ const text = typeof current.content === 'string' ? current.content : msg.content;+ const content = [{ type: 'text', text: notes.length ? `${text}\n\n(${notes.join(' ')})` : text }, ...parts];+ const out = messages.slice();+ out[idx] = { ...current, content } as T;+ return out;+}diff --git a/src/plugins/channel-bridge.ts b/src/plugins/channel-bridge.tsindex 4c8713a5..928e58d7 100644--- a/src/plugins/channel-bridge.ts+++ b/src/plugins/channel-bridge.ts@@ -170,6 +170,8 @@ export class PluginChannel extends BaseChannel {private readonly notifyOwner: (message: string) => void,/** Persist a plugin setting without restarting it (access level, members). */private readonly saveSetting: (key: string, value: unknown) => Promise<void> = async () => {},+ /** Speech to text for voice notes; null when no provider can do it. */+ private readonly transcribe: ((path: string) => Promise<string>) | null = null,) {super();this.type = manifest.contributes.channel!.type as ChannelType;@@ -389,16 +391,55 @@ export class PluginChannel extends BaseChannel {.slice(0, 10).map((a) => ({ path: a.path as string, kind: String(a.kind ?? 'document'), name: typeof a.name === 'string' ? a.name : basename(a.path as string), mime: typeof a.mime === 'string' ? a.mime : undefined })): [];- let content = text;- if (files.length) {- const list = files.map((f) => `${f.kind}: ${f.path}`).join('\n');- content = `${content}${content ? '\n' : ''}[Attachments]\n${list}`;- }- if (!content) return;+ // Voice notes become text first (in arrival order per chat), then go to the agent.+ const voice = files.filter((f) => f.kind === 'voice' || f.kind === 'audio');+ const others = files.filter((f) => !voice.includes(f));+ const chatId = p.chatId;+ const prev = this.chatQueue.get(chatId) ?? Promise.resolve();+ const next = prev.then(async () => {+ let content = text;+ let voiceNote = false;+ for (const v of voice) {+ if (!this.transcribe) {+ await this.send('🎤 I can\'t listen to voice notes yet: that needs an OpenAI provider. Add your OpenAI key once in the dashboard (Providers) or with `mercury doctor`. Meanwhile, please type it.', `${this.type}:${chatId}`).catch(() => {});+ continue;+ }+ try {+ const said = await this.transcribe(v.path);+ if (said) {+ content = `${content}${content ? '\n' : ''}${said}`;+ voiceNote = true;+ }+ } catch (err) {+ await this.send(`🎤 I couldn't understand that voice note (${(err as Error).message.slice(0, 160)}). Could you type it?`, `${this.type}:${chatId}`).catch(() => {});+ }+ }+ if (others.length) {+ const list = others.map((f) => `${f.kind}: ${f.path}`).join('\n');+ content = `${content}${content ? '\n' : ''}[Attachments: the files are attached to this message; these are their paths on this computer]\n${list}`;+ }+ if (!content) return;+ this.emitMessage(id, p, sender, role, content, others, voiceNote);+ }).catch(() => {});+ this.chatQueue.set(chatId, next);+ void next.finally(() => { if (this.chatQueue.get(chatId) === next) this.chatQueue.delete(chatId); });+ }++ private chatQueue = new Map<string, Promise<void>>();++ private emitMessage(+ id: string,+ p: IncomingParams,+ sender: string,+ role: 'admin' | 'member',+ content: string,+ files: Array<{ path: string; kind: string; name: string; mime?: string }>,+ voiceNote: boolean,+ ): void {const msg: ChannelMessage = {id,- channelId: `${this.type}:${p.chatId}`,+ channelId: `${this.type}:${p.chatId as string}`,channelType: this.type,senderId: sender,senderName: typeof p.senderName === 'string' ? p.senderName.slice(0, 100) : undefined,@@ -410,8 +451,9 @@ export class PluginChannel extends BaseChannel {chatId: p.chatId,isGroup: p.isGroup === true,attachments: files,+ voiceNote,channelAccess: this.accessFor(role),- channelGuidance: this.guidance(role, p.isGroup === true),+ channelGuidance: this.guidance(role, p.isGroup === true) + (voiceNote ? '\nThey sent this as a voice note (transcribed above); reply in text as usual.' : ''),},};this.emit(msg);diff --git a/src/plugins/channel.test.ts b/src/plugins/channel.test.tsindex 13c621ea..1b8c6cfa 100644--- a/src/plugins/channel.test.ts+++ b/src/plugins/channel.test.ts@@ -5,8 +5,9 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest';import { PermissionManager } from '../capabilities/permissions.js';import type { Channel } from '../channels/base.js';import type { ChannelMessage } from '../types/channel.js';-import { PluginManager } from './manager.js';-import { getPluginChannel, setChannelHost, splitMessage, toWhatsAppText, type PluginChannel } from './channel-bridge.js';+import { PluginManager, setProviderLookup } from './manager.js';+import { PluginChannel } from './channel-bridge.js';+import { getPluginChannel, setChannelHost, splitMessage, toWhatsAppText } from './channel-bridge.js';import { getPluginTools, resetToolBridge } from './tool-bridge.js';import { pluginDataDir } from './store.js';import { manifestSchema } from './types.js';@@ -105,6 +106,7 @@ beforeEach(() => {register: (type, channel) => { registered.push({ type, channel }); channel.onMessage((m) => received.push(m)); },unregister: (type, channel) => { registered = registered.filter((r) => r.channel !== channel); },});+ setProviderLookup(() => undefined); // never the real ~/.mercury providersmgr = new PluginManager({ safeMode: false, notify: (m) => { notices.push(m); } });});@@ -333,4 +335,48 @@ describe('plugin channels (plugin API 1.1)', () => {expect(received[2].metadata?.channelAccess).toBe('general');});});++ describe('voice notes', () => {+ function channelWith(transcribe: ((p: string) => Promise<string>) | null) {+ const sent: string[] = [];+ const got: ChannelMessage[] = [];+ const manifest = manifestSchema.parse(JSON.parse(readFileSync(join(channelPlugin(src, 'voice-notes'), 'mercury-plugin.json'), 'utf8')));+ const ch = new PluginChannel(+ manifest,+ { request: async (method, params) => { if (method === 'mercury/channel/send') sent.push((params as { text: string }).text); return {}; } },+ () => ({ owner: '+15550002', access: 'extended' }),+ () => {},+ async () => {},+ transcribe,+ );+ ch.onMessage((m) => got.push(m));+ return { ch, sent, got };+ }+ const note = (id: string, text?: string) => ({ id, chatId: '[email protected]', senderId: 'x@lid', senderPhone: '15550002', text, attachments: [{ kind: 'voice', path: `/tmp/${id}.ogg` }] });++ it('are transcribed and reach the agent as text, in order', async () => {+ const { ch, got } = channelWith(async (p) => { if (p.includes('v1')) await sleep(60); return p.includes('v1') ? 'What is the news today?' : 'And the weather?'; });+ ch.handleNotification('mercury/channel/message', note('v1'));+ ch.handleNotification('mercury/channel/message', note('v2'));+ await waitFor(() => got.length === 2);+ expect(got.map((m) => m.content)).toEqual(['What is the news today?', 'And the weather?']);+ expect(got[0].metadata?.voiceNote).toBe(true);+ expect(String(got[0].metadata?.channelGuidance)).toMatch(/voice note/);+ });++ it('without a transcriber: the owner is told how to enable it; nothing reaches the agent', async () => {+ const { ch, sent, got } = channelWith(null);+ ch.handleNotification('mercury/channel/message', note('v3'));+ await waitFor(() => sent.length === 1);+ expect(sent[0]).toMatch(/OpenAI provider[\s\S]*Providers/);+ expect(got).toEqual([]);+ });++ it('a failed transcription asks the owner to type it', async () => {+ const { ch, sent } = channelWith(async () => { throw new Error('insufficient_quota'); });+ ch.handleNotification('mercury/channel/message', note('v4'));+ await waitFor(() => sent.length === 1);+ expect(sent[0]).toMatch(/couldn't understand that voice note \(insufficient_quota\)/);+ });+ });});diff --git a/src/plugins/manager.ts b/src/plugins/manager.tsindex 27b9829c..4dbaba3d 100644--- a/src/plugins/manager.ts+++ b/src/plugins/manager.ts@@ -33,6 +33,7 @@ import { setToolOutcomeSink, unpublishPlugin, listLiveTools, listPluginCommandsimport { InProcessRuntime, inProcessLogger } from './runtime/in-process.js';import { OutOfProcessRuntime } from './runtime/out-of-process.js';import { PluginChannel, type ChannelStatus } from './channel-bridge.js';+import { transcribeAudio } from './transcribe.js';import type { PluginRuntime } from './runtime/types.js';import { providerSecretNames, type CatalogEntry, type DisabledCode, type PluginListItem, type PluginManifest, type PluginProviderId, type PluginsState } from './types.js';import { loadConfig } from '../utils/config.js';@@ -275,6 +276,11 @@ export class PluginManager {() => resolvedSettings(manifest),(m) => { void this.notifyFn(m); },(key, value) => this.setSetting(id, key, value),+ // Voice notes: transcribed with the owner's OpenAI provider, when there is one.+ (() => {+ const openai = providerLookup('openai');+ return openai?.apiKey ? (path: string) => transcribeAudio(path, { apiKey: openai.apiKey!, baseUrl: openai.baseUrl }) : null;+ })(),);}const deps = {diff --git a/src/plugins/transcribe.ts b/src/plugins/transcribe.tsnew file mode 100644index 00000000..1eac01cb--- /dev/null+++ b/src/plugins/transcribe.ts@@ -0,0 +1,37 @@+/**+ * Speech to text for voice notes, with the OpenAI provider the owner already+ * set up in Mercury (gpt-4o-mini-transcribe, falling back to whisper-1).+ */+import { readFileSync, statSync } from 'node:fs';+import { basename } from 'node:path';++const MAX_BYTES = 25 * 1024 * 1024;++export interface TranscribeCreds {+ apiKey: string;+ baseUrl?: string;+}++export async function transcribeAudio(path: string, creds: TranscribeCreds, signal?: AbortSignal): Promise<string> {+ if (statSync(path).size > MAX_BYTES) throw new Error('the voice note is longer than the 25 MB transcription limit');+ const base = (creds.baseUrl || 'https://api.openai.com/v1').replace(/\/+$/, '');+ const bytes = readFileSync(path);+ let lastError = '';+ for (const model of ['gpt-4o-mini-transcribe', 'whisper-1']) {+ const form = new FormData();+ form.append('model', model);+ form.append('file', new Blob([bytes], { type: 'audio/ogg' }), basename(path).replace(/\.[^.]+$/, '') + '.ogg');+ const res = await fetch(`${base}/audio/transcriptions`, {+ method: 'POST',+ headers: { Authorization: `Bearer ${creds.apiKey}` },+ body: form,+ signal: signal ?? AbortSignal.timeout(60_000),+ });+ const body = (await res.json().catch(() => ({}))) as { text?: string; error?: { message?: string; code?: string } };+ if (res.ok && typeof body.text === 'string') return body.text.trim();+ lastError = body.error?.message ?? `HTTP ${res.status}`;+ // Only a missing model is worth retrying with the other one.+ if (!/model/i.test(lastError) || res.status === 401 || res.status === 429) break;+ }+ throw new Error(lastError);+}