A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Store browser screenshots as attachments
Fei24116931510cc878
diff --git a/ds-bot/src/host/browser/mount.js b/ds-bot/src/host/browser/mount.jsindex f4b95cc6..31afe0a4 100644--- a/ds-bot/src/host/browser/mount.js+++ b/ds-bot/src/host/browser/mount.js@@ -14,6 +14,18 @@ export const browserOutput = {properties: {text: { type: 'string', required: true },image: { type: 'string' },+ attachment: {+ type: 'object',+ properties: {+ attachmentId: { type: 'string', required: true },+ mediaType: { type: 'string', required: true },+ bytes: { type: 'number' },+ width: { type: 'number' },+ height: { type: 'number' },+ name: { type: 'string' },+ },+ additionalProperties: true,+ },},additionalProperties: false,},@@ -22,13 +34,49 @@ export const browserOutput = {render(_args, value) {if (value && typeof value === 'object') {const blocks = [{ type: 'text', text: String(value.text ?? '') }]- if (value.image) blocks.push({ type: 'image', data: value.image, mediaType: 'image/jpeg' })+ // Image blocks must reference a stored attachment; a raw base64 payload is+ // not a valid content block and breaks the session's transport encoding.+ if (value.attachment) blocks.push({ type: 'image', attachment: value.attachment })return blocks}return [{ type: 'text', text: String(value) }]},}+function attachmentsOf(ctx) {+ try {+ const service = ctx.get?.('attachments')+ if (service !== undefined) return service+ } catch {+ // Not a cordis context, or the service was never provided.+ }+ try {+ return ctx.attachments+ } catch {+ return undefined+ }+}++// Tool bodies return screenshot bytes; the model-facing result must carry the+// stored attachment instead, since content blocks cannot inline image data.+async function saveShot(ctx, value) {+ if (!value || typeof value !== 'object' || typeof value.image !== 'string') return value+ const attachments = attachmentsOf(ctx)+ if (typeof attachments?.saveImage === 'function') {+ try {+ const attachment = await attachments.saveImage({+ mediaType: 'image/jpeg',+ data: Buffer.from(value.image, 'base64'),+ name: 'Browser screenshot',+ })+ return { text: value.text, attachment }+ } catch (error) {+ ctx.logger?.warn?.('ds-bot: screenshot could not be stored: %s', error?.message ?? error)+ }+ }+ return { text: `${value.text ?? ''} (The screenshot could not be attached.)` }+}+/*** @param {any} ctx* @param {{ toolDefs: any[], defineTool: Function, keyOf: (exec: any) => Promise<string|undefined>, profileRoot: string, downloadRoot?: string, downloadRoots?: (botId: string) => Promise<string[]>, uploadRoots?: (botId: string) => Promise<string[]>, headless: boolean, playwrightUrl?: string, executablePath?: string, actionTimeout?: number, authorize?: (exec: any, botId: string, action: any) => Promise<void>, completeFor?: (exec: any) => Function, connect?: (botId: string, downloadRoots?: string[]) => Promise<string>, getHold?: (botId: string) => any, onHold?: (botId: string, hold: any) => void, subscribeHolds?: (listener: (botId: string, hold: any) => void) => (() => void) | undefined, openSession?: typeof openBotBrowser, api?: object }} options@@ -96,12 +144,12 @@ export function mountBrowserPlugin(ctx, { toolDefs, defineTool, keyOf, profileRo}session = await sessionconst gate = authorize ? action => authorize(exec, botId, action) : undefined- return gateScope.run(gate, () => spec.execute(args ?? {}, {+ return saveShot(ctx, await gateScope.run(gate, () => spec.execute(args ?? {}, {botId,profileDir: join(profileRoot, botId),session,complete: completeFor?.(exec),- }))+ })))},}))}diff --git a/ds-bot/test/browser-plugin-output.test.js b/ds-bot/test/browser-plugin-output.test.jsindex f5d0f2b0..052c5cd2 100644--- a/ds-bot/test/browser-plugin-output.test.js+++ b/ds-bot/test/browser-plugin-output.test.js@@ -14,7 +14,10 @@ test('browser handoff and resume return valid DSH outputs and clear the Bot holdt.after(async () => ctx.fiber.dispose())let hold = null+ const ref = { attachmentId: 'sha256:abcd', mediaType: 'image/jpeg', bytes: 3, width: 1, height: 1 }+ const saved = []const plugin = inner => {+ inner.provide('attachments', { saveImage: async input => { saved.push(input); return ref } })mountBrowserPlugin(inner, {toolDefs: refsTools.filter(tool => ['browser_handoff', 'browser_resume', 'browser_screenshot'].includes(tool.name)),defineTool,@@ -70,10 +73,50 @@ test('browser handoff and resume return valid DSH outputs and clear the Bot holdconst screenshot = await execute('browser_screenshot', {})assert.equal(screenshot.isError, false, screenshot.error?.message)- assert.deepEqual(screenshot.value, { text: 'Screenshot of https://example.com/.', image: 'AAAA' })+ assert.deepEqual(saved, [{+ mediaType: 'image/jpeg',+ data: Buffer.from('AAAA', 'base64'),+ name: 'Browser screenshot',+ }])+ assert.deepEqual(screenshot.value, { text: 'Screenshot of https://example.com/.', attachment: ref })assert.deepEqual(screenshot.content, [{ type: 'text', text: 'Screenshot of https://example.com/.' },- { type: 'image', data: 'AAAA', mediaType: 'image/jpeg' },+ { type: 'image', attachment: ref },+ ])+})++test('a screenshot degrades to text when no attachments service is mounted', async t => {+ const ctx = new Context()+ await ctx.plugin(SystemPrompt)+ await ctx.plugin(ToolRuntime)+ t.after(async () => ctx.fiber.dispose())++ const plugin = inner => {+ mountBrowserPlugin(inner, {+ toolDefs: refsTools.filter(tool => tool.name === 'browser_screenshot'),+ defineTool,+ keyOf: async () => 'bot-1',+ profileRoot: '/tmp/browser-profiles',+ headless: true,+ openSession: async () => ({+ async shot() { return { text: 'Screenshot of https://example.com/.', image: 'AAAA' } },+ async close() {},+ }),+ })+ }+ plugin.inject = ['tools']+ await ctx.plugin(plugin)++ const screenshot = await ctx.tools.execute({+ signal: new AbortController().signal,+ callId: ToolCallId('browser_screenshot'),+ name: 'browser_screenshot',+ arguments: {},+ })+ assert.equal(screenshot.isError, false, screenshot.error?.message)+ assert.deepEqual(screenshot.value, { text: 'Screenshot of https://example.com/. (The screenshot could not be attached.)' })+ assert.deepEqual(screenshot.content, [+ { type: 'text', text: 'Screenshot of https://example.com/. (The screenshot could not be attached.)' },])})