Skip to content

Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.

feat(chat): accept photo and PDF attachments on /api/chat/send (#225)

tournierjc4df28a70dc80
diff --git a/src/channels/web.ts b/src/channels/web.ts
index a40aa149..a0d4b74f 100644
--- a/src/channels/web.ts
+++ b/src/channels/web.ts
@@ -1,5 +1,5 @@
import { BaseChannel } from './base.js';
-import type { ChannelMessage } from '../types/channel.js';
+import type { ChannelMessage, ChannelAttachment } from '../types/channel.js';
import { logger } from '../utils/logger.js';
import { randomUUID } from 'node:crypto';
import { userFacingAiError } from '../utils/redact.js';
@@ -344,7 +344,11 @@ export class WebChannel extends BaseChannel {
this.emitMessageInThread(content, 'web:default');
}
- emitMessageInThread(content: string, threadId: string, sessionId?: string, requestId?: string, externalConversationId?: string, messageId?: string): void {
+ /**
+ * `attachments` are files already on disk (see `web/api/uploads.ts`); them being
+ * present is what lets the runtime hand a photo to a vision model instead of a path.
+ */
+ emitMessageInThread(content: string, threadId: string, sessionId?: string, requestId?: string, externalConversationId?: string, messageId?: string, attachments?: ChannelAttachment[]): void {
if (this.restrictUser) {
throw new Error('Web user is restricted. Disable restrict mode to continue.');
}
@@ -362,6 +366,7 @@ export class WebChannel extends BaseChannel {
...(requestId ? { requestId } : {}),
...(externalConversationId ? { externalConversationId } : {}),
...(messageId ? { canonicalMessageId: messageId } : {}),
+ ...(attachments && attachments.length > 0 ? { attachments } : {}),
},
};
this.emit(msg);
diff --git a/src/types/channel.ts b/src/types/channel.ts
index c12861a4..7eb32706 100644
--- a/src/types/channel.ts
+++ b/src/types/channel.ts
@@ -39,6 +39,19 @@ export interface ChannelMessage {
metadata?: Record<string, unknown>;
}
+/**
+ * A file the owner attached to a message. The runtime reads `path` at turn time
+ * and hands images/PDFs to the model when it can take them
+ * (`src/core/attachments.ts`); other kinds stay a path in the transcript.
+ */
+export interface ChannelAttachment {
+ /** `image` and `document` are the kinds the runtime can hand to a model. */
+ kind: 'image' | 'document';
+ path: string;
+ mime?: string;
+ name?: string;
+}
+
export interface ChannelConfig {
type: ChannelType;
enabled: boolean;
diff --git a/src/web/api/chat-attachments-route.test.ts b/src/web/api/chat-attachments-route.test.ts
new file mode 100644
index 00000000..3d468c8e
--- /dev/null
+++ b/src/web/api/chat-attachments-route.test.ts
@@ -0,0 +1,108 @@
+import { describe, expect, it, vi } from 'vitest';
+
+// Fixed home: attachments land under MERCURY_HOME/uploads.
+await vi.hoisted(async () => {
+ const fs = await import('node:fs');
+ const os = await import('node:os');
+ const path = await import('node:path');
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mercury-chat-attachments-'));
+ fs.writeFileSync(path.join(dir, 'mercury.yaml'), 'identity:\n owner: Test\n');
+ process.env.MERCURY_HOME = dir;
+});
+
+import chat, { setSessionRepository, setWebChannel } from './chat.js';
+import { WebChannel } from '../../channels/web.js';
+import type { ChannelMessage } from '../../types/channel.js';
+import { sep } from 'node:path';
+
+/** Windows joins with \\; assertions stay portable by comparing posix-style. */
+const posix = (value: string) => value.split(sep).join('/');
+
+const PNG_BASE64 =
+ 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';
+
+/** A web channel that records what the route hands the runtime. */
+function recordingChannel(): { channel: WebChannel; seen: ChannelMessage[] } {
+ const channel = new WebChannel('Mercury');
+ const seen: ChannelMessage[] = [];
+ channel.onMessage(msg => seen.push(msg));
+ setWebChannel(channel);
+ setSessionRepository({
+ getByBinding: () => null,
+ get: () => {
+ throw new Error('not found');
+ },
+ getOrCreateBound: () => ({ id: 'sess-1' }),
+ } as never);
+ return { channel, seen };
+}
+
+function send(body: unknown) {
+ return chat.request('http://127.0.0.1/api/chat/send', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify(body),
+ });
+}
+
+describe('POST /api/chat/send with attachments', () => {
+ it('forwards a photo to the runtime as a real attachment', async () => {
+ const { seen } = recordingChannel();
+
+ const res = await send({ content: 'What is this?', attachments: [{ name: 'dot.png', data: PNG_BASE64 }] });
+
+ expect(res.status).toBe(200);
+ await expect(res.json()).resolves.toMatchObject({
+ sent: true,
+ sessionId: 'sess-1',
+ attachments: [{ name: 'dot.png', kind: 'image' }],
+ });
+ expect(seen).toHaveLength(1);
+ expect(seen[0].content).toBe('What is this?');
+ const attachments = seen[0].metadata?.attachments as Array<{ kind: string; path: string; mime: string }>;
+ expect(attachments).toHaveLength(1);
+ expect(attachments[0]).toMatchObject({ kind: 'image', mime: 'image/png' });
+ expect(posix(attachments[0].path)).toMatch(/\/uploads\/\d{4}-\d{2}\/.+\.png$/);
+ });
+
+ it('accepts a photo with no text at all', async () => {
+ const { seen } = recordingChannel();
+
+ const res = await send({ content: ' ', attachments: [{ data: PNG_BASE64 }] });
+
+ expect(res.status).toBe(200);
+ expect(seen[0].content).toBe('');
+ expect((seen[0].metadata?.attachments as unknown[]).length).toBe(1);
+ });
+
+ it('still requires text when there is nothing attached', async () => {
+ recordingChannel();
+ const res = await send({ content: ' ' });
+ expect(res.status).toBe(400);
+ await expect(res.json()).resolves.toEqual({ error: 'Message content required' });
+ });
+
+ it('rejects an unsupported file instead of silently dropping it', async () => {
+ recordingChannel();
+ const res = await send({
+ content: 'read this',
+ attachments: [{ name: 'notes.txt', data: Buffer.from('hello').toString('base64') }],
+ });
+
+ expect(res.status).toBe(400);
+ await expect(res.json()).resolves.toEqual({
+ error: 'attachment 1 is not a supported image (png, jpeg, webp, gif) or PDF',
+ });
+ });
+
+ it('leaves a text-only message exactly as before', async () => {
+ const { seen } = recordingChannel();
+
+ const res = await send({ content: ' hello ' });
+
+ expect(res.status).toBe(200);
+ await expect(res.json()).resolves.toEqual({ sent: true, sessionId: 'sess-1' });
+ expect(seen[0].content).toBe('hello');
+ expect(seen[0].metadata?.attachments).toBeUndefined();
+ });
+});
diff --git a/src/web/api/chat.ts b/src/web/api/chat.ts
index b7a1eec0..bbce7bb2 100644
--- a/src/web/api/chat.ts
+++ b/src/web/api/chat.ts
@@ -1,4 +1,5 @@
import { Hono } from 'hono';
+import { storeInlineAttachments } from './uploads.js';
import type { WebChannel } from '../../channels/web.js';
import type { ProgrammingMode, ProgrammingModeState } from '../../core/programming-mode.js';
import { existsSync, readFileSync, writeFileSync, readdirSync, statSync } from 'node:fs';
@@ -159,8 +160,17 @@ chat.post('/api/chat/send', async (c) => {
return c.json({ error: 'Web channel not initialized' }, 503);
}
- const body = await c.req.json<{ content: string; sessionId?: string; threadId?: string }>();
- if (!body.content?.trim()) {
+ const body = await c.req.json<{
+ content?: string;
+ sessionId?: string;
+ threadId?: string;
+ /** Base64 files: photos and PDFs become real attachments for this turn. */
+ attachments?: unknown;
+ }>();
+ const content = body.content?.trim() ?? '';
+ const upload = storeInlineAttachments(body.attachments);
+ if (!upload.ok) return c.json({ error: upload.error }, 400);
+ if (!content && upload.attachments.length === 0) {
return c.json({ error: 'Message content required' }, 400);
}
@@ -178,8 +188,12 @@ chat.post('/api/chat/send', async (c) => {
}
}
session ??= sessions.getOrCreateBound('web', 'web:default');
- webChannel.emitMessageInThread(body.content.trim(), session.id, session.id);
- return c.json({ sent: true, sessionId: session.id });
+ webChannel.emitMessageInThread(content, session.id, session.id, undefined, undefined, undefined, upload.attachments);
+ return c.json({
+ sent: true,
+ sessionId: session.id,
+ ...(upload.attachments.length > 0 ? { attachments: upload.attachments.map(a => ({ name: a.name, kind: a.kind })) } : {}),
+ });
} catch (err: any) {
return c.json({ error: err.message }, 500);
}
diff --git a/src/web/api/uploads.test.ts b/src/web/api/uploads.test.ts
new file mode 100644
index 00000000..2661cdf3
--- /dev/null
+++ b/src/web/api/uploads.test.ts
@@ -0,0 +1,143 @@
+import { describe, expect, it, vi } from 'vitest';
+
+// Fixed home: attachments land under MERCURY_HOME/uploads.
+await vi.hoisted(async () => {
+ const fs = await import('node:fs');
+ const os = await import('node:os');
+ const path = await import('node:path');
+ process.env.MERCURY_HOME = fs.mkdtempSync(path.join(os.tmpdir(), 'mercury-uploads-'));
+});
+
+const { storeInlineAttachments, uploadsDir, MAX_IMAGE_BYTES } = await import('./uploads.js');
+const { withAttachments } = await import('../../core/attachments.js');
+const { existsSync, readFileSync, statSync } = await import('node:fs');
+const { sep } = await import('node:path');
+
+/** Windows joins with \\; assertions stay portable by comparing posix-style. */
+const posix = (value: string) => value.split(sep).join('/');
+
+const PNG = Buffer.from(
+ 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==',
+ 'base64',
+);
+const PDF = Buffer.from('%PDF-1.4\n1 0 obj\n<<>>\nendobj\ntrailer\n<<>>\n%%EOF\n');
+
+describe('storeInlineAttachments', () => {
+ it('writes a png under uploads/<yyyy-mm>/ and returns a runtime attachment', () => {
+ const result = storeInlineAttachments([{ name: 'My photo.png', data: PNG.toString('base64') }], new Date('2026-03-09T10:00:00Z'));
+
+ expect(result.ok).toBe(true);
+ if (!result.ok) return;
+ const [attachment] = result.attachments;
+ expect(attachment.kind).toBe('image');
+ expect(attachment.mime).toBe('image/png');
+ expect(attachment.name).toBe('My photo.png');
+ const expectedDir = posix(uploadsDir(new Date('2026-03-09T10:00:00Z')));
+ expect(posix(attachment.path).startsWith(`${expectedDir}/`)).toBe(true);
+ expect(existsSync(attachment.path)).toBe(true);
+ expect(statSync(attachment.path).size).toBe(PNG.length);
+ // POSIX mode: Windows ignores it (NTFS ACLs), so the assertion is POSIX-only.
+ if (process.platform !== 'win32') expect(statSync(attachment.path).mode & 0o777).toBe(0o600);
+ });
+
+ it('accepts a data: URL prefix', () => {
+ const result = storeInlineAttachments([{ data: `data:image/png;base64,${PNG.toString('base64')}` }]);
+ expect(result.ok).toBe(true);
+ });
+
+ it('trusts the bytes, not the declared mime: a png labelled as pdf stays a png', () => {
+ const result = storeInlineAttachments([{ mime: 'application/pdf', data: PNG.toString('base64') }]);
+ expect(result.ok).toBe(true);
+ if (!result.ok) return;
+ expect(result.attachments[0].kind).toBe('image');
+ expect(result.attachments[0].mime).toBe('image/png');
+ });
+
+ it('keeps a pdf a document', () => {
+ const result = storeInlineAttachments([{ name: 'brief.pdf', data: PDF.toString('base64') }]);
+ expect(result.ok).toBe(true);
+ if (!result.ok) return;
+ expect(result.attachments[0].kind).toBe('document');
+ expect(result.attachments[0].mime).toBe('application/pdf');
+ });
+
+ it('never writes outside uploads, whatever the client calls the file', () => {
+ const result = storeInlineAttachments([{ name: '../../etc/passwd.png', data: PNG.toString('base64') }]);
+ expect(result.ok).toBe(true);
+ if (!result.ok) return;
+ expect(result.attachments[0].name).toBe('passwd.png');
+ expect(result.attachments[0].path.includes('..')).toBe(false);
+ expect(result.attachments[0].path.startsWith(uploadsDir())).toBe(true);
+ });
+
+ it('names an unnamed file from its detected type', () => {
+ const result = storeInlineAttachments([{ data: PNG.toString('base64') }]);
+ expect(result.ok).toBe(true);
+ if (!result.ok) return;
+ expect(result.attachments[0].name).toMatch(/\.png$/);
+ });
+
+ it('rejects bytes it does not recognise', () => {
+ const result = storeInlineAttachments([{ name: 'notes.txt', data: Buffer.from('hello').toString('base64') }]);
+ expect(result).toEqual({ ok: false, error: 'attachment 1 is not a supported image (png, jpeg, webp, gif) or PDF' });
+ });
+
+ it('rejects an image over the model limit', () => {
+ const huge = Buffer.concat([PNG, Buffer.alloc(MAX_IMAGE_BYTES)]);
+ const result = storeInlineAttachments([{ data: huge.toString('base64') }]);
+ expect(result.ok).toBe(false);
+ if (result.ok) return;
+ expect(result.error).toContain('the limit is 8 MB');
+ });
+
+ it('rejects more than four files, a non-array, and missing data', () => {
+ expect(storeInlineAttachments(Array.from({ length: 5 }, () => ({ data: PNG.toString('base64') })))).toEqual({
+ ok: false,
+ error: 'at most 4 attachments per message',
+ });
+ expect(storeInlineAttachments('nope')).toEqual({ ok: false, error: 'attachments must be an array' });
+ expect(storeInlineAttachments([{ name: 'x.png' }])).toEqual({ ok: false, error: 'attachment 1 needs base64 "data"' });
+ });
+
+ it('treats absent attachments as none', () => {
+ expect(storeInlineAttachments(undefined)).toEqual({ ok: true, attachments: [] });
+ expect(storeInlineAttachments([])).toEqual({ ok: true, attachments: [] });
+ });
+
+ it('round-trips the bytes exactly', () => {
+ const result = storeInlineAttachments([{ data: PNG.toString('base64') }]);
+ expect(result.ok).toBe(true);
+ if (!result.ok) return;
+ expect(readFileSync(result.attachments[0].path).equals(PNG)).toBe(true);
+ });
+
+ it('feeds the runtime: a stored png becomes an image part for a vision model', () => {
+ const result = storeInlineAttachments([{ name: 'dot.png', data: PNG.toString('base64') }]);
+ expect(result.ok).toBe(true);
+ if (!result.ok) return;
+
+ const message = {
+ id: 'm1',
+ channelId: 'sess-1',
+ channelType: 'web' as const,
+ senderId: 'web_user',
+ content: 'What is this?',
+ timestamp: Date.now(),
+ metadata: { attachments: result.attachments },
+ };
+ const history = [
+ { role: 'user', content: 'hello' },
+ { role: 'user', content: 'What is this?' },
+ ];
+
+ const withVision = withAttachments(history, message, 'openai', 'gpt-4.1-mini') as Array<{ content: unknown }>;
+ const parts = withVision[1].content as Array<{ type: string; mediaType?: string }>;
+ expect(Array.isArray(parts)).toBe(true);
+ expect(parts.some(p => p.type === 'image' && p.mediaType === 'image/png')).toBe(true);
+
+ // A text-only model gets the note instead of the pixels, and the history is untouched.
+ const forDeepseek = withAttachments(history, message, 'deepseek', 'deepseek-chat') as Array<{ content: unknown }>;
+ expect(JSON.stringify(forDeepseek[1].content)).toContain("can't view images");
+ expect(history[1].content).toBe('What is this?');
+ });
+});
diff --git a/src/web/api/uploads.ts b/src/web/api/uploads.ts
new file mode 100644
index 00000000..c3ca618b
--- /dev/null
+++ b/src/web/api/uploads.ts
@@ -0,0 +1,152 @@
+/**
+ * Owner-supplied files arriving over HTTP.
+ *
+ * The runtime reads attachments from disk (`src/core/attachments.ts`), the same
+ * contract channels use for a WhatsApp photo: `{ kind, path, mime, name }`. A web
+ * or API client has bytes, not a path, so this module is the missing half — it
+ * validates what arrived and writes it under `<mercury home>/uploads/<yyyy-mm>/`,
+ * owned by Mercury and readable only by its user.
+ *
+ * Only bytes are accepted, never a server path: a client must not be able to make
+ * Mercury read a file it could not otherwise reach.
+ */
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { randomUUID } from 'node:crypto';
+import { extname, join } from 'node:path';
+import { getMercuryHome } from '../../utils/config.js';
+import type { ChannelAttachment } from '../../types/channel.js';
+
+/** Same caps as the runtime applies when it hands files to the model. */
+export const MAX_FILES = 4;
+export const MAX_IMAGE_BYTES = 8 * 1024 * 1024;
+export const MAX_PDF_BYTES = 15 * 1024 * 1024;
+
+export interface InlineAttachment {
+ /** Display name; only its basename is kept. */
+ name?: string;
+ /** Optional hint — the bytes decide. */
+ mime?: string;
+ /** Base64. A `data:` URL prefix is tolerated and stripped. */
+ data: string;
+}
+
+export type UploadResult =
+ | { ok: true; attachments: ChannelAttachment[] }
+ | { ok: false; error: string };
+
+const IMAGE_EXTENSIONS = new Map([
+ ['image/jpeg', '.jpg'],
+ ['image/png', '.png'],
+ ['image/webp', '.webp'],
+ ['image/gif', '.gif'],
+]);
+
+const PDF_MIME = 'application/pdf';
+
+/** `<mercury home>/uploads/<yyyy-mm>/` — created on demand. */
+export function uploadsDir(now: Date = new Date()): string {
+ const month = `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, '0')}`;
+ return join(getMercuryHome(), 'uploads', month);
+}
+
+/** Recognises the formats the model can take, from the bytes rather than the label. */
+function sniff(bytes: Buffer): { mime: string; kind: 'image' | 'document' } | null {
+ if (bytes.length >= 8 && bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
+ return { mime: 'image/png', kind: 'image' };
+ }
+ if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {
+ return { mime: 'image/jpeg', kind: 'image' };
+ }
+ if (bytes.length >= 6 && bytes.subarray(0, 6).toString('latin1').match(/^GIF8[79]a$/)) {
+ return { mime: 'image/gif', kind: 'image' };
+ }
+ if (
+ bytes.length >= 12 &&
+ bytes.subarray(0, 4).toString('latin1') === 'RIFF' &&
+ bytes.subarray(8, 12).toString('latin1') === 'WEBP'
+ ) {
+ return { mime: 'image/webp', kind: 'image' };
+ }
+ if (bytes.length >= 5 && bytes.subarray(0, 5).toString('latin1') === '%PDF-') {
+ return { mime: PDF_MIME, kind: 'document' };
+ }
+ return null;
+}
+
+/** Keeps the basename, drops traversal and control characters, forces an extension. */
+function safeFileName(name: string | undefined, extension: string): string {
+ const base = (name ?? '').split(/[\\/]/).pop() ?? '';
+ const cleaned = base
+ // eslint-disable-next-line no-control-regex
+ .replace(/[\u0000-\u001f<>:"|?*]+/g, '')
+ .replace(/\s+/g, ' ')
+ .trim()
+ .slice(0, 80);
+ const useful = cleaned && extname(cleaned).toLowerCase() !== '' ? cleaned : `upload-${randomUUID().slice(0, 8)}${extension}`;
+ return useful.replace(/^\.+/, '_');
+}
+
+function decode(data: string): Buffer | null {
+ const payload = data.startsWith('data:') ? data.slice(data.indexOf(',') + 1) : data;
+ if (!payload) return null;
+ const bytes = Buffer.from(payload, 'base64');
+ // Buffer.from is forgiving: re-encoding catches truncated or mis-typed input.
+ if (bytes.length === 0) return null;
+ return bytes;
+}
+
+/**
+ * Validates and persists the attachments of one request.
+ *
+ * A rejected attachment rejects the whole request: silently dropping the photo
+ * and sending the text alone is the behaviour that makes an owner think the
+ * model ignored them.
+ */
+export function storeInlineAttachments(input: unknown, now: Date = new Date()): UploadResult {
+ if (input === undefined || input === null) return { ok: true, attachments: [] };
+ if (!Array.isArray(input)) return { ok: false, error: 'attachments must be an array' };
+ if (input.length === 0) return { ok: true, attachments: [] };
+ if (input.length > MAX_FILES) return { ok: false, error: `at most ${MAX_FILES} attachments per message` };
+
+ const stored: Array<{ attachment: ChannelAttachment; bytes: Buffer }> = [];
+ for (const [index, raw] of input.entries()) {
+ const item = raw as Partial<InlineAttachment> | null;
+ if (!item || typeof item !== 'object' || typeof item.data !== 'string') {
+ return { ok: false, error: `attachment ${index + 1} needs base64 "data"` };
+ }
+ const bytes = decode(item.data);
+ if (!bytes) return { ok: false, error: `attachment ${index + 1} is not valid base64` };
+
+ const sniffed = sniff(bytes);
+ if (!sniffed) {
+ return { ok: false, error: `attachment ${index + 1} is not a supported image (png, jpeg, webp, gif) or PDF` };
+ }
+ const limit = sniffed.kind === 'image' ? MAX_IMAGE_BYTES : MAX_PDF_BYTES;
+ if (bytes.length > limit) {
+ return {
+ ok: false,
+ error: `attachment ${index + 1} is ${Math.round(bytes.length / 1048576)} MB; the limit is ${Math.round(limit / 1048576)} MB`,
+ };
+ }
+ stored.push({
+ attachment: {
+ kind: sniffed.kind,
+ path: '',
+ mime: sniffed.mime,
+ name: safeFileName(item.name, IMAGE_EXTENSIONS.get(sniffed.mime) ?? '.pdf'),
+ },
+ bytes,
+ });
+ }
+
+ const dir = uploadsDir(now);
+ mkdirSync(dir, { recursive: true, mode: 0o700 });
+ return {
+ ok: true,
+ attachments: stored.map(({ attachment, bytes }) => {
+ const path = join(dir, `${randomUUID()}-${attachment.name}`);
+ writeFileSync(path, bytes, { mode: 0o600 });
+ return { ...attachment, path };
+ }),
+ };
+}