Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

Store browser screenshots as attachments

Fei24116931510cc878
diff --git a/ds-bot/src/host/browser/mount.js b/ds-bot/src/host/browser/mount.js
index f4b95cc6..31afe0a4 100644
--- a/ds-bot/src/host/browser/mount.js
+++ b/ds-bot/src/host/browser/mount.js
@@ -14,6 +14,18 @@ export const browserOutput = {
properties: {
text: { type: 'string', required: true },
image: { type: 'string' },
+ attachment: {
+ type: 'object',
+ properties: {
+ attachmentId: { type: 'string', required: true },
+ mediaType: { type: 'string', required: true },
+ bytes: { type: 'number' },
+ width: { type: 'number' },
+ height: { type: 'number' },
+ name: { type: 'string' },
+ },
+ additionalProperties: true,
+ },
},
additionalProperties: false,
},
@@ -22,13 +34,49 @@ export const browserOutput = {
render(_args, value) {
if (value && typeof value === 'object') {
const blocks = [{ type: 'text', text: String(value.text ?? '') }]
- if (value.image) blocks.push({ type: 'image', data: value.image, mediaType: 'image/jpeg' })
+ // Image blocks must reference a stored attachment; a raw base64 payload is
+ // not a valid content block and breaks the session's transport encoding.
+ if (value.attachment) blocks.push({ type: 'image', attachment: value.attachment })
return blocks
}
return [{ type: 'text', text: String(value) }]
},
}
+function attachmentsOf(ctx) {
+ try {
+ const service = ctx.get?.('attachments')
+ if (service !== undefined) return service
+ } catch {
+ // Not a cordis context, or the service was never provided.
+ }
+ try {
+ return ctx.attachments
+ } catch {
+ return undefined
+ }
+}
+
+// Tool bodies return screenshot bytes; the model-facing result must carry the
+// stored attachment instead, since content blocks cannot inline image data.
+async function saveShot(ctx, value) {
+ if (!value || typeof value !== 'object' || typeof value.image !== 'string') return value
+ const attachments = attachmentsOf(ctx)
+ if (typeof attachments?.saveImage === 'function') {
+ try {
+ const attachment = await attachments.saveImage({
+ mediaType: 'image/jpeg',
+ data: Buffer.from(value.image, 'base64'),
+ name: 'Browser screenshot',
+ })
+ return { text: value.text, attachment }
+ } catch (error) {
+ ctx.logger?.warn?.('ds-bot: screenshot could not be stored: %s', error?.message ?? error)
+ }
+ }
+ return { text: `${value.text ?? ''} (The screenshot could not be attached.)` }
+}
+
/**
* @param {any} ctx
* @param {{ toolDefs: any[], defineTool: Function, keyOf: (exec: any) => Promise<string|undefined>, profileRoot: string, downloadRoot?: string, downloadRoots?: (botId: string) => Promise<string[]>, uploadRoots?: (botId: string) => Promise<string[]>, headless: boolean, playwrightUrl?: string, executablePath?: string, actionTimeout?: number, authorize?: (exec: any, botId: string, action: any) => Promise<void>, completeFor?: (exec: any) => Function, connect?: (botId: string, downloadRoots?: string[]) => Promise<string>, getHold?: (botId: string) => any, onHold?: (botId: string, hold: any) => void, subscribeHolds?: (listener: (botId: string, hold: any) => void) => (() => void) | undefined, openSession?: typeof openBotBrowser, api?: object }} options
@@ -96,12 +144,12 @@ export function mountBrowserPlugin(ctx, { toolDefs, defineTool, keyOf, profileRo
}
session = await session
const gate = authorize ? action => authorize(exec, botId, action) : undefined
- return gateScope.run(gate, () => spec.execute(args ?? {}, {
+ return saveShot(ctx, await gateScope.run(gate, () => spec.execute(args ?? {}, {
botId,
profileDir: join(profileRoot, botId),
session,
complete: completeFor?.(exec),
- }))
+ })))
},
}))
}
diff --git a/ds-bot/test/browser-plugin-output.test.js b/ds-bot/test/browser-plugin-output.test.js
index f5d0f2b0..052c5cd2 100644
--- a/ds-bot/test/browser-plugin-output.test.js
+++ b/ds-bot/test/browser-plugin-output.test.js
@@ -14,7 +14,10 @@ test('browser handoff and resume return valid DSH outputs and clear the Bot hold
t.after(async () => ctx.fiber.dispose())
let hold = null
+ const ref = { attachmentId: 'sha256:abcd', mediaType: 'image/jpeg', bytes: 3, width: 1, height: 1 }
+ const saved = []
const plugin = inner => {
+ inner.provide('attachments', { saveImage: async input => { saved.push(input); return ref } })
mountBrowserPlugin(inner, {
toolDefs: refsTools.filter(tool => ['browser_handoff', 'browser_resume', 'browser_screenshot'].includes(tool.name)),
defineTool,
@@ -70,10 +73,50 @@ test('browser handoff and resume return valid DSH outputs and clear the Bot hold
const screenshot = await execute('browser_screenshot', {})
assert.equal(screenshot.isError, false, screenshot.error?.message)
- assert.deepEqual(screenshot.value, { text: 'Screenshot of https://example.com/.', image: 'AAAA' })
+ assert.deepEqual(saved, [{
+ mediaType: 'image/jpeg',
+ data: Buffer.from('AAAA', 'base64'),
+ name: 'Browser screenshot',
+ }])
+ assert.deepEqual(screenshot.value, { text: 'Screenshot of https://example.com/.', attachment: ref })
assert.deepEqual(screenshot.content, [
{ type: 'text', text: 'Screenshot of https://example.com/.' },
- { type: 'image', data: 'AAAA', mediaType: 'image/jpeg' },
+ { type: 'image', attachment: ref },
+ ])
+})
+
+test('a screenshot degrades to text when no attachments service is mounted', async t => {
+ const ctx = new Context()
+ await ctx.plugin(SystemPrompt)
+ await ctx.plugin(ToolRuntime)
+ t.after(async () => ctx.fiber.dispose())
+
+ const plugin = inner => {
+ mountBrowserPlugin(inner, {
+ toolDefs: refsTools.filter(tool => tool.name === 'browser_screenshot'),
+ defineTool,
+ keyOf: async () => 'bot-1',
+ profileRoot: '/tmp/browser-profiles',
+ headless: true,
+ openSession: async () => ({
+ async shot() { return { text: 'Screenshot of https://example.com/.', image: 'AAAA' } },
+ async close() {},
+ }),
+ })
+ }
+ plugin.inject = ['tools']
+ await ctx.plugin(plugin)
+
+ const screenshot = await ctx.tools.execute({
+ signal: new AbortController().signal,
+ callId: ToolCallId('browser_screenshot'),
+ name: 'browser_screenshot',
+ arguments: {},
+ })
+ assert.equal(screenshot.isError, false, screenshot.error?.message)
+ assert.deepEqual(screenshot.value, { text: 'Screenshot of https://example.com/. (The screenshot could not be attached.)' })
+ assert.deepEqual(screenshot.content, [
+ { type: 'text', text: 'Screenshot of https://example.com/. (The screenshot could not be attached.)' },
])
})