Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.
feat(chat): accept photo and PDF attachments on /api/chat/send (#225)
tournierjc4df28a70dc80
diff --git a/src/channels/web.ts b/src/channels/web.tsindex a40aa149..a0d4b74f 100644--- a/src/channels/web.ts+++ b/src/channels/web.ts@@ -1,5 +1,5 @@import { BaseChannel } from './base.js';-import type { ChannelMessage } from '../types/channel.js';+import type { ChannelMessage, ChannelAttachment } from '../types/channel.js';import { logger } from '../utils/logger.js';import { randomUUID } from 'node:crypto';import { userFacingAiError } from '../utils/redact.js';@@ -344,7 +344,11 @@ export class WebChannel extends BaseChannel {this.emitMessageInThread(content, 'web:default');}- emitMessageInThread(content: string, threadId: string, sessionId?: string, requestId?: string, externalConversationId?: string, messageId?: string): void {+ /**+ * `attachments` are files already on disk (see `web/api/uploads.ts`); them being+ * present is what lets the runtime hand a photo to a vision model instead of a path.+ */+ emitMessageInThread(content: string, threadId: string, sessionId?: string, requestId?: string, externalConversationId?: string, messageId?: string, attachments?: ChannelAttachment[]): void {if (this.restrictUser) {throw new Error('Web user is restricted. Disable restrict mode to continue.');}@@ -362,6 +366,7 @@ export class WebChannel extends BaseChannel {...(requestId ? { requestId } : {}),...(externalConversationId ? { externalConversationId } : {}),...(messageId ? { canonicalMessageId: messageId } : {}),+ ...(attachments && attachments.length > 0 ? { attachments } : {}),},};this.emit(msg);diff --git a/src/types/channel.ts b/src/types/channel.tsindex c12861a4..7eb32706 100644--- a/src/types/channel.ts+++ b/src/types/channel.ts@@ -39,6 +39,19 @@ export interface ChannelMessage {metadata?: Record<string, unknown>;}+/**+ * A file the owner attached to a message. The runtime reads `path` at turn time+ * and hands images/PDFs to the model when it can take them+ * (`src/core/attachments.ts`); other kinds stay a path in the transcript.+ */+export interface ChannelAttachment {+ /** `image` and `document` are the kinds the runtime can hand to a model. */+ kind: 'image' | 'document';+ path: string;+ mime?: string;+ name?: string;+}+export interface ChannelConfig {type: ChannelType;enabled: boolean;diff --git a/src/web/api/chat-attachments-route.test.ts b/src/web/api/chat-attachments-route.test.tsnew file mode 100644index 00000000..3d468c8e--- /dev/null+++ b/src/web/api/chat-attachments-route.test.ts@@ -0,0 +1,108 @@+import { describe, expect, it, vi } from 'vitest';++// Fixed home: attachments land under MERCURY_HOME/uploads.+await vi.hoisted(async () => {+ const fs = await import('node:fs');+ const os = await import('node:os');+ const path = await import('node:path');+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mercury-chat-attachments-'));+ fs.writeFileSync(path.join(dir, 'mercury.yaml'), 'identity:\n owner: Test\n');+ process.env.MERCURY_HOME = dir;+});++import chat, { setSessionRepository, setWebChannel } from './chat.js';+import { WebChannel } from '../../channels/web.js';+import type { ChannelMessage } from '../../types/channel.js';+import { sep } from 'node:path';++/** Windows joins with \\; assertions stay portable by comparing posix-style. */+const posix = (value: string) => value.split(sep).join('/');++const PNG_BASE64 =+ 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';++/** A web channel that records what the route hands the runtime. */+function recordingChannel(): { channel: WebChannel; seen: ChannelMessage[] } {+ const channel = new WebChannel('Mercury');+ const seen: ChannelMessage[] = [];+ channel.onMessage(msg => seen.push(msg));+ setWebChannel(channel);+ setSessionRepository({+ getByBinding: () => null,+ get: () => {+ throw new Error('not found');+ },+ getOrCreateBound: () => ({ id: 'sess-1' }),+ } as never);+ return { channel, seen };+}++function send(body: unknown) {+ return chat.request('http://127.0.0.1/api/chat/send', {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify(body),+ });+}++describe('POST /api/chat/send with attachments', () => {+ it('forwards a photo to the runtime as a real attachment', async () => {+ const { seen } = recordingChannel();++ const res = await send({ content: 'What is this?', attachments: [{ name: 'dot.png', data: PNG_BASE64 }] });++ expect(res.status).toBe(200);+ await expect(res.json()).resolves.toMatchObject({+ sent: true,+ sessionId: 'sess-1',+ attachments: [{ name: 'dot.png', kind: 'image' }],+ });+ expect(seen).toHaveLength(1);+ expect(seen[0].content).toBe('What is this?');+ const attachments = seen[0].metadata?.attachments as Array<{ kind: string; path: string; mime: string }>;+ expect(attachments).toHaveLength(1);+ expect(attachments[0]).toMatchObject({ kind: 'image', mime: 'image/png' });+ expect(posix(attachments[0].path)).toMatch(/\/uploads\/\d{4}-\d{2}\/.+\.png$/);+ });++ it('accepts a photo with no text at all', async () => {+ const { seen } = recordingChannel();++ const res = await send({ content: ' ', attachments: [{ data: PNG_BASE64 }] });++ expect(res.status).toBe(200);+ expect(seen[0].content).toBe('');+ expect((seen[0].metadata?.attachments as unknown[]).length).toBe(1);+ });++ it('still requires text when there is nothing attached', async () => {+ recordingChannel();+ const res = await send({ content: ' ' });+ expect(res.status).toBe(400);+ await expect(res.json()).resolves.toEqual({ error: 'Message content required' });+ });++ it('rejects an unsupported file instead of silently dropping it', async () => {+ recordingChannel();+ const res = await send({+ content: 'read this',+ attachments: [{ name: 'notes.txt', data: Buffer.from('hello').toString('base64') }],+ });++ expect(res.status).toBe(400);+ await expect(res.json()).resolves.toEqual({+ error: 'attachment 1 is not a supported image (png, jpeg, webp, gif) or PDF',+ });+ });++ it('leaves a text-only message exactly as before', async () => {+ const { seen } = recordingChannel();++ const res = await send({ content: ' hello ' });++ expect(res.status).toBe(200);+ await expect(res.json()).resolves.toEqual({ sent: true, sessionId: 'sess-1' });+ expect(seen[0].content).toBe('hello');+ expect(seen[0].metadata?.attachments).toBeUndefined();+ });+});diff --git a/src/web/api/chat.ts b/src/web/api/chat.tsindex b7a1eec0..bbce7bb2 100644--- a/src/web/api/chat.ts+++ b/src/web/api/chat.ts@@ -1,4 +1,5 @@import { Hono } from 'hono';+import { storeInlineAttachments } from './uploads.js';import type { WebChannel } from '../../channels/web.js';import type { ProgrammingMode, ProgrammingModeState } from '../../core/programming-mode.js';import { existsSync, readFileSync, writeFileSync, readdirSync, statSync } from 'node:fs';@@ -159,8 +160,17 @@ chat.post('/api/chat/send', async (c) => {return c.json({ error: 'Web channel not initialized' }, 503);}- const body = await c.req.json<{ content: string; sessionId?: string; threadId?: string }>();- if (!body.content?.trim()) {+ const body = await c.req.json<{+ content?: string;+ sessionId?: string;+ threadId?: string;+ /** Base64 files: photos and PDFs become real attachments for this turn. */+ attachments?: unknown;+ }>();+ const content = body.content?.trim() ?? '';+ const upload = storeInlineAttachments(body.attachments);+ if (!upload.ok) return c.json({ error: upload.error }, 400);+ if (!content && upload.attachments.length === 0) {return c.json({ error: 'Message content required' }, 400);}@@ -178,8 +188,12 @@ chat.post('/api/chat/send', async (c) => {}}session ??= sessions.getOrCreateBound('web', 'web:default');- webChannel.emitMessageInThread(body.content.trim(), session.id, session.id);- return c.json({ sent: true, sessionId: session.id });+ webChannel.emitMessageInThread(content, session.id, session.id, undefined, undefined, undefined, upload.attachments);+ return c.json({+ sent: true,+ sessionId: session.id,+ ...(upload.attachments.length > 0 ? { attachments: upload.attachments.map(a => ({ name: a.name, kind: a.kind })) } : {}),+ });} catch (err: any) {return c.json({ error: err.message }, 500);}diff --git a/src/web/api/uploads.test.ts b/src/web/api/uploads.test.tsnew file mode 100644index 00000000..2661cdf3--- /dev/null+++ b/src/web/api/uploads.test.ts@@ -0,0 +1,143 @@+import { describe, expect, it, vi } from 'vitest';++// Fixed home: attachments land under MERCURY_HOME/uploads.+await vi.hoisted(async () => {+ const fs = await import('node:fs');+ const os = await import('node:os');+ const path = await import('node:path');+ process.env.MERCURY_HOME = fs.mkdtempSync(path.join(os.tmpdir(), 'mercury-uploads-'));+});++const { storeInlineAttachments, uploadsDir, MAX_IMAGE_BYTES } = await import('./uploads.js');+const { withAttachments } = await import('../../core/attachments.js');+const { existsSync, readFileSync, statSync } = await import('node:fs');+const { sep } = await import('node:path');++/** Windows joins with \\; assertions stay portable by comparing posix-style. */+const posix = (value: string) => value.split(sep).join('/');++const PNG = Buffer.from(+ 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==',+ 'base64',+);+const PDF = Buffer.from('%PDF-1.4\n1 0 obj\n<<>>\nendobj\ntrailer\n<<>>\n%%EOF\n');++describe('storeInlineAttachments', () => {+ it('writes a png under uploads/<yyyy-mm>/ and returns a runtime attachment', () => {+ const result = storeInlineAttachments([{ name: 'My photo.png', data: PNG.toString('base64') }], new Date('2026-03-09T10:00:00Z'));++ expect(result.ok).toBe(true);+ if (!result.ok) return;+ const [attachment] = result.attachments;+ expect(attachment.kind).toBe('image');+ expect(attachment.mime).toBe('image/png');+ expect(attachment.name).toBe('My photo.png');+ const expectedDir = posix(uploadsDir(new Date('2026-03-09T10:00:00Z')));+ expect(posix(attachment.path).startsWith(`${expectedDir}/`)).toBe(true);+ expect(existsSync(attachment.path)).toBe(true);+ expect(statSync(attachment.path).size).toBe(PNG.length);+ // POSIX mode: Windows ignores it (NTFS ACLs), so the assertion is POSIX-only.+ if (process.platform !== 'win32') expect(statSync(attachment.path).mode & 0o777).toBe(0o600);+ });++ it('accepts a data: URL prefix', () => {+ const result = storeInlineAttachments([{ data: `data:image/png;base64,${PNG.toString('base64')}` }]);+ expect(result.ok).toBe(true);+ });++ it('trusts the bytes, not the declared mime: a png labelled as pdf stays a png', () => {+ const result = storeInlineAttachments([{ mime: 'application/pdf', data: PNG.toString('base64') }]);+ expect(result.ok).toBe(true);+ if (!result.ok) return;+ expect(result.attachments[0].kind).toBe('image');+ expect(result.attachments[0].mime).toBe('image/png');+ });++ it('keeps a pdf a document', () => {+ const result = storeInlineAttachments([{ name: 'brief.pdf', data: PDF.toString('base64') }]);+ expect(result.ok).toBe(true);+ if (!result.ok) return;+ expect(result.attachments[0].kind).toBe('document');+ expect(result.attachments[0].mime).toBe('application/pdf');+ });++ it('never writes outside uploads, whatever the client calls the file', () => {+ const result = storeInlineAttachments([{ name: '../../etc/passwd.png', data: PNG.toString('base64') }]);+ expect(result.ok).toBe(true);+ if (!result.ok) return;+ expect(result.attachments[0].name).toBe('passwd.png');+ expect(result.attachments[0].path.includes('..')).toBe(false);+ expect(result.attachments[0].path.startsWith(uploadsDir())).toBe(true);+ });++ it('names an unnamed file from its detected type', () => {+ const result = storeInlineAttachments([{ data: PNG.toString('base64') }]);+ expect(result.ok).toBe(true);+ if (!result.ok) return;+ expect(result.attachments[0].name).toMatch(/\.png$/);+ });++ it('rejects bytes it does not recognise', () => {+ const result = storeInlineAttachments([{ name: 'notes.txt', data: Buffer.from('hello').toString('base64') }]);+ expect(result).toEqual({ ok: false, error: 'attachment 1 is not a supported image (png, jpeg, webp, gif) or PDF' });+ });++ it('rejects an image over the model limit', () => {+ const huge = Buffer.concat([PNG, Buffer.alloc(MAX_IMAGE_BYTES)]);+ const result = storeInlineAttachments([{ data: huge.toString('base64') }]);+ expect(result.ok).toBe(false);+ if (result.ok) return;+ expect(result.error).toContain('the limit is 8 MB');+ });++ it('rejects more than four files, a non-array, and missing data', () => {+ expect(storeInlineAttachments(Array.from({ length: 5 }, () => ({ data: PNG.toString('base64') })))).toEqual({+ ok: false,+ error: 'at most 4 attachments per message',+ });+ expect(storeInlineAttachments('nope')).toEqual({ ok: false, error: 'attachments must be an array' });+ expect(storeInlineAttachments([{ name: 'x.png' }])).toEqual({ ok: false, error: 'attachment 1 needs base64 "data"' });+ });++ it('treats absent attachments as none', () => {+ expect(storeInlineAttachments(undefined)).toEqual({ ok: true, attachments: [] });+ expect(storeInlineAttachments([])).toEqual({ ok: true, attachments: [] });+ });++ it('round-trips the bytes exactly', () => {+ const result = storeInlineAttachments([{ data: PNG.toString('base64') }]);+ expect(result.ok).toBe(true);+ if (!result.ok) return;+ expect(readFileSync(result.attachments[0].path).equals(PNG)).toBe(true);+ });++ it('feeds the runtime: a stored png becomes an image part for a vision model', () => {+ const result = storeInlineAttachments([{ name: 'dot.png', data: PNG.toString('base64') }]);+ expect(result.ok).toBe(true);+ if (!result.ok) return;++ const message = {+ id: 'm1',+ channelId: 'sess-1',+ channelType: 'web' as const,+ senderId: 'web_user',+ content: 'What is this?',+ timestamp: Date.now(),+ metadata: { attachments: result.attachments },+ };+ const history = [+ { role: 'user', content: 'hello' },+ { role: 'user', content: 'What is this?' },+ ];++ const withVision = withAttachments(history, message, 'openai', 'gpt-4.1-mini') as Array<{ content: unknown }>;+ const parts = withVision[1].content as Array<{ type: string; mediaType?: string }>;+ expect(Array.isArray(parts)).toBe(true);+ expect(parts.some(p => p.type === 'image' && p.mediaType === 'image/png')).toBe(true);++ // A text-only model gets the note instead of the pixels, and the history is untouched.+ const forDeepseek = withAttachments(history, message, 'deepseek', 'deepseek-chat') as Array<{ content: unknown }>;+ expect(JSON.stringify(forDeepseek[1].content)).toContain("can't view images");+ expect(history[1].content).toBe('What is this?');+ });+});diff --git a/src/web/api/uploads.ts b/src/web/api/uploads.tsnew file mode 100644index 00000000..c3ca618b--- /dev/null+++ b/src/web/api/uploads.ts@@ -0,0 +1,152 @@+/**+ * Owner-supplied files arriving over HTTP.+ *+ * The runtime reads attachments from disk (`src/core/attachments.ts`), the same+ * contract channels use for a WhatsApp photo: `{ kind, path, mime, name }`. A web+ * or API client has bytes, not a path, so this module is the missing half — it+ * validates what arrived and writes it under `<mercury home>/uploads/<yyyy-mm>/`,+ * owned by Mercury and readable only by its user.+ *+ * Only bytes are accepted, never a server path: a client must not be able to make+ * Mercury read a file it could not otherwise reach.+ */+import { mkdirSync, writeFileSync } from 'node:fs';+import { randomUUID } from 'node:crypto';+import { extname, join } from 'node:path';+import { getMercuryHome } from '../../utils/config.js';+import type { ChannelAttachment } from '../../types/channel.js';++/** Same caps as the runtime applies when it hands files to the model. */+export const MAX_FILES = 4;+export const MAX_IMAGE_BYTES = 8 * 1024 * 1024;+export const MAX_PDF_BYTES = 15 * 1024 * 1024;++export interface InlineAttachment {+ /** Display name; only its basename is kept. */+ name?: string;+ /** Optional hint — the bytes decide. */+ mime?: string;+ /** Base64. A `data:` URL prefix is tolerated and stripped. */+ data: string;+}++export type UploadResult =+ | { ok: true; attachments: ChannelAttachment[] }+ | { ok: false; error: string };++const IMAGE_EXTENSIONS = new Map([+ ['image/jpeg', '.jpg'],+ ['image/png', '.png'],+ ['image/webp', '.webp'],+ ['image/gif', '.gif'],+]);++const PDF_MIME = 'application/pdf';++/** `<mercury home>/uploads/<yyyy-mm>/` — created on demand. */+export function uploadsDir(now: Date = new Date()): string {+ const month = `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, '0')}`;+ return join(getMercuryHome(), 'uploads', month);+}++/** Recognises the formats the model can take, from the bytes rather than the label. */+function sniff(bytes: Buffer): { mime: string; kind: 'image' | 'document' } | null {+ if (bytes.length >= 8 && bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {+ return { mime: 'image/png', kind: 'image' };+ }+ if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {+ return { mime: 'image/jpeg', kind: 'image' };+ }+ if (bytes.length >= 6 && bytes.subarray(0, 6).toString('latin1').match(/^GIF8[79]a$/)) {+ return { mime: 'image/gif', kind: 'image' };+ }+ if (+ bytes.length >= 12 &&+ bytes.subarray(0, 4).toString('latin1') === 'RIFF' &&+ bytes.subarray(8, 12).toString('latin1') === 'WEBP'+ ) {+ return { mime: 'image/webp', kind: 'image' };+ }+ if (bytes.length >= 5 && bytes.subarray(0, 5).toString('latin1') === '%PDF-') {+ return { mime: PDF_MIME, kind: 'document' };+ }+ return null;+}++/** Keeps the basename, drops traversal and control characters, forces an extension. */+function safeFileName(name: string | undefined, extension: string): string {+ const base = (name ?? '').split(/[\\/]/).pop() ?? '';+ const cleaned = base+ // eslint-disable-next-line no-control-regex+ .replace(/[\u0000-\u001f<>:"|?*]+/g, '')+ .replace(/\s+/g, ' ')+ .trim()+ .slice(0, 80);+ const useful = cleaned && extname(cleaned).toLowerCase() !== '' ? cleaned : `upload-${randomUUID().slice(0, 8)}${extension}`;+ return useful.replace(/^\.+/, '_');+}++function decode(data: string): Buffer | null {+ const payload = data.startsWith('data:') ? data.slice(data.indexOf(',') + 1) : data;+ if (!payload) return null;+ const bytes = Buffer.from(payload, 'base64');+ // Buffer.from is forgiving: re-encoding catches truncated or mis-typed input.+ if (bytes.length === 0) return null;+ return bytes;+}++/**+ * Validates and persists the attachments of one request.+ *+ * A rejected attachment rejects the whole request: silently dropping the photo+ * and sending the text alone is the behaviour that makes an owner think the+ * model ignored them.+ */+export function storeInlineAttachments(input: unknown, now: Date = new Date()): UploadResult {+ if (input === undefined || input === null) return { ok: true, attachments: [] };+ if (!Array.isArray(input)) return { ok: false, error: 'attachments must be an array' };+ if (input.length === 0) return { ok: true, attachments: [] };+ if (input.length > MAX_FILES) return { ok: false, error: `at most ${MAX_FILES} attachments per message` };++ const stored: Array<{ attachment: ChannelAttachment; bytes: Buffer }> = [];+ for (const [index, raw] of input.entries()) {+ const item = raw as Partial<InlineAttachment> | null;+ if (!item || typeof item !== 'object' || typeof item.data !== 'string') {+ return { ok: false, error: `attachment ${index + 1} needs base64 "data"` };+ }+ const bytes = decode(item.data);+ if (!bytes) return { ok: false, error: `attachment ${index + 1} is not valid base64` };++ const sniffed = sniff(bytes);+ if (!sniffed) {+ return { ok: false, error: `attachment ${index + 1} is not a supported image (png, jpeg, webp, gif) or PDF` };+ }+ const limit = sniffed.kind === 'image' ? MAX_IMAGE_BYTES : MAX_PDF_BYTES;+ if (bytes.length > limit) {+ return {+ ok: false,+ error: `attachment ${index + 1} is ${Math.round(bytes.length / 1048576)} MB; the limit is ${Math.round(limit / 1048576)} MB`,+ };+ }+ stored.push({+ attachment: {+ kind: sniffed.kind,+ path: '',+ mime: sniffed.mime,+ name: safeFileName(item.name, IMAGE_EXTENSIONS.get(sniffed.mime) ?? '.pdf'),+ },+ bytes,+ });+ }++ const dir = uploadsDir(now);+ mkdirSync(dir, { recursive: true, mode: 0o700 });+ return {+ ok: true,+ attachments: stored.map(({ attachment, bytes }) => {+ const path = join(dir, `${randomUUID()}-${attachment.name}`);+ writeFileSync(path, bytes, { mode: 0o600 });+ return { ...attachment, path };+ }),+ };+}