Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(switchdash): surface non-image attachments to dash-managed sessions (CHOO-1802) (#94)

Louis Amaudruzfec776545553
diff --git a/dash/apps/switchdash-desktop/src/main/core/switch-rooms/room-connection.test.ts b/dash/apps/switchdash-desktop/src/main/core/switch-rooms/room-connection.test.ts
index 43a3079bb..6c564fc27 100644
--- a/dash/apps/switchdash-desktop/src/main/core/switch-rooms/room-connection.test.ts
+++ b/dash/apps/switchdash-desktop/src/main/core/switch-rooms/room-connection.test.ts
@@ -178,13 +178,16 @@ describe('RoomConnection', () => {
fs.rmSync(mediaDir, { recursive: true, force: true });
});
- it('does not annotate or download non-image attachments', async () => {
+ it('downloads and annotates non-image attachments', async () => {
+ fs.rmSync(mediaDir, { recursive: true, force: true });
const target: InjectionTarget = { write: vi.fn() };
+ // A structural-biology file: an arbitrary mimetype the app has never heard
+ // of must still reach the agent.
const attachment: AttachmentRef = {
- filename: 'notes.pdf',
- mimetype: 'application/pdf',
+ filename: 'tyk2_ejm_31_minimized.pdb',
+ mimetype: 'chemical/x-pdb',
size: 3,
- mxc: 'mxc://switch.test/pdf',
+ mxc: 'mxc://switch.test/pdb',
msgtype: 'm.file',
};
const { conn, fetchMock } = connect({ acquire: () => target }, [
@@ -193,11 +196,57 @@ describe('RoomConnection', () => {
await flush();
- expect(fetchMock.mock.calls.filter((c) => String(c[0]).includes('/media')).length).toBe(0);
+ const mediaCalls = fetchMock.mock.calls.filter((c) => String(c[0]).includes('/media'));
+ expect(mediaCalls.length).toBe(1);
+ expect(String(mediaCalls[0][0])).toContain(encodeURIComponent(attachment.mxc));
+
const injected = vi.mocked(target.write).mock.calls.map((c) => c[0])[0];
- expect(injected).not.toContain('attached');
+ expect(injected).toContain('1 file attached');
+ expect(injected).toContain(mediaDir);
conn.stop();
+ fs.rmSync(mediaDir, { recursive: true, force: true });
+ });
+
+ it('annotates images and other files separately in one message', async () => {
+ fs.rmSync(mediaDir, { recursive: true, force: true });
+ const target: InjectionTarget = { write: vi.fn() };
+ const attachments: AttachmentRef[] = [
+ {
+ filename: 'shot.png',
+ mimetype: 'image/png',
+ size: 3,
+ mxc: 'mxc://switch.test/png',
+ msgtype: 'm.image',
+ },
+ {
+ filename: 'notes.md',
+ mimetype: 'text/markdown',
+ size: 3,
+ mxc: 'mxc://switch.test/md',
+ msgtype: 'm.file',
+ },
+ {
+ filename: 'data.csv',
+ mimetype: 'text/csv',
+ size: 3,
+ mxc: 'mxc://switch.test/csv',
+ msgtype: 'm.file',
+ },
+ ];
+ const { conn, fetchMock } = connect({ acquire: () => target }, [
+ messageEvent(true, null, attachments),
+ ]);
+
+ await flush();
+
+ expect(fetchMock.mock.calls.filter((c) => String(c[0]).includes('/media')).length).toBe(3);
+ const injected = vi.mocked(target.write).mock.calls.map((c) => c[0])[0];
+ expect(injected).toContain('1 image attached');
+ expect(injected).toContain('2 files attached');
+
+ conn.stop();
+ fs.rmSync(mediaDir, { recursive: true, force: true });
});
it('does not inject unaddressed chatter', async () => {
diff --git a/dash/apps/switchdash-desktop/src/main/core/switch-rooms/room-connection.ts b/dash/apps/switchdash-desktop/src/main/core/switch-rooms/room-connection.ts
index 3d735b27f..510ea6da3 100644
--- a/dash/apps/switchdash-desktop/src/main/core/switch-rooms/room-connection.ts
+++ b/dash/apps/switchdash-desktop/src/main/core/switch-rooms/room-connection.ts
@@ -5,7 +5,7 @@ import type { InjectionSink } from './injection-sink';
import type { SessionControl } from './session-control';
import {
formatEventForInjection,
- formatImageAttachmentAnnotation,
+ formatAttachmentAnnotation,
type AgentBridgeEventResponse,
type AttachmentRef,
type CommandPayload,
@@ -530,15 +530,17 @@ export class RoomConnection {
if (text) {
let annotated = text;
if (event.type === 'message') {
- // Materialise image attachments to local files and tell the agent
- // they are there — parity with the connector channel, which the
- // pollers otherwise lacked. Best-effort: a failed download is
- // logged and omitted, never blocking the message.
- const imagePaths = await this.downloadImageAttachments(
+ // Materialise attachments of ANY type to local files and tell the
+ // agent they are there — parity with the connector channel, which
+ // the pollers otherwise lacked. A download that fails is named in
+ // the annotation rather than dropped, so the agent is never left
+ // believing it saw everything.
+ const { imagePaths, filePaths, failed } = await this.downloadAttachments(
event.payload as MessagePayload
);
- if (imagePaths.length > 0) {
- annotated = `${annotated}\n${formatImageAttachmentAnnotation(imagePaths)}`;
+ const annotation = formatAttachmentAnnotation(imagePaths, filePaths, failed);
+ if (annotation) {
+ annotated = `${annotated}\n${annotation}`;
}
}
const body =
@@ -562,23 +564,34 @@ export class RoomConnection {
}
/**
- * Download a message's image attachments to local files under `mediaDir`,
- * returning their paths. Mirrors the connector channel: filter by an
- * `image/*` mimetype, fetch the bytes from the agent bridge (which proxies
- * the Matrix media repo), and write them locally so the agent can Read them.
- * Best-effort — a failed download is logged and skipped, never thrown, so it
- * cannot break event delivery.
+ * Download a message's attachments — of any type — to local files under
+ * `mediaDir`. Mirrors the connector channel: fetch the bytes from the agent
+ * bridge (which proxies the Matrix media repo) and write them locally so the
+ * agent can Read them.
+ *
+ * Images and other files are returned separately so the annotation can
+ * describe each appropriately. A download that fails returns its filename in
+ * `failed` instead of being dropped: it is surfaced to the agent, and it
+ * still never throws, so it cannot break event delivery.
*/
- private async downloadImageAttachments(msg: MessagePayload): Promise<string[]> {
+ private async downloadAttachments(
+ msg: MessagePayload
+ ): Promise<{ imagePaths: string[]; filePaths: string[]; failed: string[] }> {
const attachments = msg.attachments ?? [];
- const paths: string[] = [];
+ const imagePaths: string[] = [];
+ const filePaths: string[] = [];
+ const failed: string[] = [];
for (let i = 0; i < attachments.length; i++) {
const att = attachments[i];
- if (!att.mimetype.startsWith('image/')) continue;
const localPath = await this.fetchAttachmentToFile(att, msg.message_id, i);
- if (localPath) paths.push(localPath);
+ if (!localPath) {
+ failed.push(att.filename);
+ continue;
+ }
+ if (att.mimetype.startsWith('image/')) imagePaths.push(localPath);
+ else filePaths.push(localPath);
}
- return paths;
+ return { imagePaths, filePaths, failed };
}
private async fetchAttachmentToFile(
diff --git a/dash/apps/switchdash-desktop/src/main/core/switch-rooms/switch-event-format.ts b/dash/apps/switchdash-desktop/src/main/core/switch-rooms/switch-event-format.ts
index 5d172c899..9b386dfd0 100644
--- a/dash/apps/switchdash-desktop/src/main/core/switch-rooms/switch-event-format.ts
+++ b/dash/apps/switchdash-desktop/src/main/core/switch-rooms/switch-event-format.ts
@@ -140,17 +140,48 @@ export function formatEventForInjection(
}
/**
- * Annotation appended to an injected message that carried image attachments.
- * Unlike the connector channel — which surfaces images as an `image_path`
- * notification attribute plus a static instruction — the pollers inject plain
- * text into a PTY/tmux pane and have no separate metadata channel, so the
- * annotation itself must both signal the attachments and tell the agent how to
- * view them. `localPaths` are the local files the images were downloaded to.
+ * Annotation appended to an injected message that carried attachments.
+ * Unlike the connector channel — which surfaces them as `image_path` /
+ * `file_path` notification attributes plus a static instruction — the pollers
+ * inject plain text into a PTY/tmux pane and have no separate metadata
+ * channel, so the annotation itself must both signal the attachments and tell
+ * the agent how to read them.
+ *
+ * Images and other files are named separately because the agent treats them
+ * differently (an image is Read to be seen; a `.md`/`.csv`/`.pdb` is Read to be
+ * parsed). `failed` names attachments that could not be downloaded: they are
+ * reported rather than omitted, so the agent never silently believes it saw
+ * everything the sender attached.
*/
-export function formatImageAttachmentAnnotation(localPaths: string[]): string {
- const count = localPaths.length;
- const noun = count === 1 ? 'image' : 'images';
- return `(${count} ${noun} attached — downloaded to ${localPaths.join(', ')}. Read ${
- count === 1 ? 'it' : 'them'
- } to view.)`;
+export function formatAttachmentAnnotation(
+ imagePaths: string[],
+ filePaths: string[],
+ failed: string[] = []
+): string | null {
+ const parts: string[] = [];
+ if (imagePaths.length > 0) {
+ const noun = imagePaths.length === 1 ? 'image' : 'images';
+ parts.push(
+ `${imagePaths.length} ${noun} attached — downloaded to ${imagePaths.join(', ')}. Read ${
+ imagePaths.length === 1 ? 'it' : 'them'
+ } to view.`
+ );
+ }
+ if (filePaths.length > 0) {
+ const noun = filePaths.length === 1 ? 'file' : 'files';
+ parts.push(
+ `${filePaths.length} ${noun} attached — downloaded to ${filePaths.join(', ')}. Read ${
+ filePaths.length === 1 ? 'it' : 'them'
+ } to see the contents.`
+ );
+ }
+ if (failed.length > 0) {
+ parts.push(
+ `${failed.length} attachment${failed.length === 1 ? '' : 's'} could NOT be retrieved (${failed.join(
+ ', '
+ )}) — do not assume you have seen ${failed.length === 1 ? 'it' : 'them'}.`
+ );
+ }
+ if (parts.length === 0) return null;
+ return `(${parts.join(' ')})`;
}