Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
fix(switchdash): surface non-image attachments to dash-managed sessions (CHOO-1802) (#94)
Louis Amaudruzfec776545553
diff --git a/dash/apps/switchdash-desktop/src/main/core/switch-rooms/room-connection.test.ts b/dash/apps/switchdash-desktop/src/main/core/switch-rooms/room-connection.test.tsindex 43a3079bb..6c564fc27 100644--- a/dash/apps/switchdash-desktop/src/main/core/switch-rooms/room-connection.test.ts+++ b/dash/apps/switchdash-desktop/src/main/core/switch-rooms/room-connection.test.ts@@ -178,13 +178,16 @@ describe('RoomConnection', () => {fs.rmSync(mediaDir, { recursive: true, force: true });});- it('does not annotate or download non-image attachments', async () => {+ it('downloads and annotates non-image attachments', async () => {+ fs.rmSync(mediaDir, { recursive: true, force: true });const target: InjectionTarget = { write: vi.fn() };+ // A structural-biology file: an arbitrary mimetype the app has never heard+ // of must still reach the agent.const attachment: AttachmentRef = {- filename: 'notes.pdf',- mimetype: 'application/pdf',+ filename: 'tyk2_ejm_31_minimized.pdb',+ mimetype: 'chemical/x-pdb',size: 3,- mxc: 'mxc://switch.test/pdf',+ mxc: 'mxc://switch.test/pdb',msgtype: 'm.file',};const { conn, fetchMock } = connect({ acquire: () => target }, [@@ -193,11 +196,57 @@ describe('RoomConnection', () => {await flush();- expect(fetchMock.mock.calls.filter((c) => String(c[0]).includes('/media')).length).toBe(0);+ const mediaCalls = fetchMock.mock.calls.filter((c) => String(c[0]).includes('/media'));+ expect(mediaCalls.length).toBe(1);+ expect(String(mediaCalls[0][0])).toContain(encodeURIComponent(attachment.mxc));+const injected = vi.mocked(target.write).mock.calls.map((c) => c[0])[0];- expect(injected).not.toContain('attached');+ expect(injected).toContain('1 file attached');+ expect(injected).toContain(mediaDir);conn.stop();+ fs.rmSync(mediaDir, { recursive: true, force: true });+ });++ it('annotates images and other files separately in one message', async () => {+ fs.rmSync(mediaDir, { recursive: true, force: true });+ const target: InjectionTarget = { write: vi.fn() };+ const attachments: AttachmentRef[] = [+ {+ filename: 'shot.png',+ mimetype: 'image/png',+ size: 3,+ mxc: 'mxc://switch.test/png',+ msgtype: 'm.image',+ },+ {+ filename: 'notes.md',+ mimetype: 'text/markdown',+ size: 3,+ mxc: 'mxc://switch.test/md',+ msgtype: 'm.file',+ },+ {+ filename: 'data.csv',+ mimetype: 'text/csv',+ size: 3,+ mxc: 'mxc://switch.test/csv',+ msgtype: 'm.file',+ },+ ];+ const { conn, fetchMock } = connect({ acquire: () => target }, [+ messageEvent(true, null, attachments),+ ]);++ await flush();++ expect(fetchMock.mock.calls.filter((c) => String(c[0]).includes('/media')).length).toBe(3);+ const injected = vi.mocked(target.write).mock.calls.map((c) => c[0])[0];+ expect(injected).toContain('1 image attached');+ expect(injected).toContain('2 files attached');++ conn.stop();+ fs.rmSync(mediaDir, { recursive: true, force: true });});it('does not inject unaddressed chatter', async () => {diff --git a/dash/apps/switchdash-desktop/src/main/core/switch-rooms/room-connection.ts b/dash/apps/switchdash-desktop/src/main/core/switch-rooms/room-connection.tsindex 3d735b27f..510ea6da3 100644--- a/dash/apps/switchdash-desktop/src/main/core/switch-rooms/room-connection.ts+++ b/dash/apps/switchdash-desktop/src/main/core/switch-rooms/room-connection.ts@@ -5,7 +5,7 @@ import type { InjectionSink } from './injection-sink';import type { SessionControl } from './session-control';import {formatEventForInjection,- formatImageAttachmentAnnotation,+ formatAttachmentAnnotation,type AgentBridgeEventResponse,type AttachmentRef,type CommandPayload,@@ -530,15 +530,17 @@ export class RoomConnection {if (text) {let annotated = text;if (event.type === 'message') {- // Materialise image attachments to local files and tell the agent- // they are there — parity with the connector channel, which the- // pollers otherwise lacked. Best-effort: a failed download is- // logged and omitted, never blocking the message.- const imagePaths = await this.downloadImageAttachments(+ // Materialise attachments of ANY type to local files and tell the+ // agent they are there — parity with the connector channel, which+ // the pollers otherwise lacked. A download that fails is named in+ // the annotation rather than dropped, so the agent is never left+ // believing it saw everything.+ const { imagePaths, filePaths, failed } = await this.downloadAttachments(event.payload as MessagePayload);- if (imagePaths.length > 0) {- annotated = `${annotated}\n${formatImageAttachmentAnnotation(imagePaths)}`;+ const annotation = formatAttachmentAnnotation(imagePaths, filePaths, failed);+ if (annotation) {+ annotated = `${annotated}\n${annotation}`;}}const body =@@ -562,23 +564,34 @@ export class RoomConnection {}/**- * Download a message's image attachments to local files under `mediaDir`,- * returning their paths. Mirrors the connector channel: filter by an- * `image/*` mimetype, fetch the bytes from the agent bridge (which proxies- * the Matrix media repo), and write them locally so the agent can Read them.- * Best-effort — a failed download is logged and skipped, never thrown, so it- * cannot break event delivery.+ * Download a message's attachments — of any type — to local files under+ * `mediaDir`. Mirrors the connector channel: fetch the bytes from the agent+ * bridge (which proxies the Matrix media repo) and write them locally so the+ * agent can Read them.+ *+ * Images and other files are returned separately so the annotation can+ * describe each appropriately. A download that fails returns its filename in+ * `failed` instead of being dropped: it is surfaced to the agent, and it+ * still never throws, so it cannot break event delivery.*/- private async downloadImageAttachments(msg: MessagePayload): Promise<string[]> {+ private async downloadAttachments(+ msg: MessagePayload+ ): Promise<{ imagePaths: string[]; filePaths: string[]; failed: string[] }> {const attachments = msg.attachments ?? [];- const paths: string[] = [];+ const imagePaths: string[] = [];+ const filePaths: string[] = [];+ const failed: string[] = [];for (let i = 0; i < attachments.length; i++) {const att = attachments[i];- if (!att.mimetype.startsWith('image/')) continue;const localPath = await this.fetchAttachmentToFile(att, msg.message_id, i);- if (localPath) paths.push(localPath);+ if (!localPath) {+ failed.push(att.filename);+ continue;+ }+ if (att.mimetype.startsWith('image/')) imagePaths.push(localPath);+ else filePaths.push(localPath);}- return paths;+ return { imagePaths, filePaths, failed };}private async fetchAttachmentToFile(diff --git a/dash/apps/switchdash-desktop/src/main/core/switch-rooms/switch-event-format.ts b/dash/apps/switchdash-desktop/src/main/core/switch-rooms/switch-event-format.tsindex 5d172c899..9b386dfd0 100644--- a/dash/apps/switchdash-desktop/src/main/core/switch-rooms/switch-event-format.ts+++ b/dash/apps/switchdash-desktop/src/main/core/switch-rooms/switch-event-format.ts@@ -140,17 +140,48 @@ export function formatEventForInjection(}/**- * Annotation appended to an injected message that carried image attachments.- * Unlike the connector channel — which surfaces images as an `image_path`- * notification attribute plus a static instruction — the pollers inject plain- * text into a PTY/tmux pane and have no separate metadata channel, so the- * annotation itself must both signal the attachments and tell the agent how to- * view them. `localPaths` are the local files the images were downloaded to.+ * Annotation appended to an injected message that carried attachments.+ * Unlike the connector channel — which surfaces them as `image_path` /+ * `file_path` notification attributes plus a static instruction — the pollers+ * inject plain text into a PTY/tmux pane and have no separate metadata+ * channel, so the annotation itself must both signal the attachments and tell+ * the agent how to read them.+ *+ * Images and other files are named separately because the agent treats them+ * differently (an image is Read to be seen; a `.md`/`.csv`/`.pdb` is Read to be+ * parsed). `failed` names attachments that could not be downloaded: they are+ * reported rather than omitted, so the agent never silently believes it saw+ * everything the sender attached.*/-export function formatImageAttachmentAnnotation(localPaths: string[]): string {- const count = localPaths.length;- const noun = count === 1 ? 'image' : 'images';- return `(${count} ${noun} attached — downloaded to ${localPaths.join(', ')}. Read ${- count === 1 ? 'it' : 'them'- } to view.)`;+export function formatAttachmentAnnotation(+ imagePaths: string[],+ filePaths: string[],+ failed: string[] = []+): string | null {+ const parts: string[] = [];+ if (imagePaths.length > 0) {+ const noun = imagePaths.length === 1 ? 'image' : 'images';+ parts.push(+ `${imagePaths.length} ${noun} attached — downloaded to ${imagePaths.join(', ')}. Read ${+ imagePaths.length === 1 ? 'it' : 'them'+ } to view.`+ );+ }+ if (filePaths.length > 0) {+ const noun = filePaths.length === 1 ? 'file' : 'files';+ parts.push(+ `${filePaths.length} ${noun} attached — downloaded to ${filePaths.join(', ')}. Read ${+ filePaths.length === 1 ? 'it' : 'them'+ } to see the contents.`+ );+ }+ if (failed.length > 0) {+ parts.push(+ `${failed.length} attachment${failed.length === 1 ? '' : 's'} could NOT be retrieved (${failed.join(+ ', '+ )}) — do not assume you have seen ${failed.length === 1 ? 'it' : 'them'}.`+ );+ }+ if (parts.length === 0) return null;+ return `(${parts.join(' ')})`;}