Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

Repair persisted browser screenshots before the next model step

Fei24114b8138215abc
diff --cc ds-bot/src/host/browser/mount.js
index 31afe0a4,f4b95cc6..0fc58b21
--- a/ds-bot/src/host/browser/mount.js
+++ b/ds-bot/src/host/browser/mount.js
@@@ -43,46 -29,12 +43,62 @@@ export const browserOutput =
},
}
- function attachmentsOf(ctx) {
- try {
- const service = ctx.get?.('attachments')
- if (service !== undefined) return service
- } catch {
- // Not a cordis context, or the service was never provided.
- }
- try {
- return ctx.attachments
- } catch {
- return undefined
- }
- }
-
+// Tool bodies return screenshot bytes; the model-facing result must carry the
+// stored attachment instead, since content blocks cannot inline image data.
+async function saveShot(ctx, value) {
+ if (!value || typeof value !== 'object' || typeof value.image !== 'string') return value
- const attachments = attachmentsOf(ctx)
++ const attachments = ctx.get('attachments')
+ if (typeof attachments?.saveImage === 'function') {
+ try {
+ const attachment = await attachments.saveImage({
+ mediaType: 'image/jpeg',
+ data: Buffer.from(value.image, 'base64'),
+ name: 'Browser screenshot',
+ })
+ return { text: value.text, attachment }
+ } catch (error) {
+ ctx.logger?.warn?.('ds-bot: screenshot could not be stored: %s', error?.message ?? error)
+ }
+ }
+ return { text: `${value.text ?? ''} (The screenshot could not be attached.)` }
+}
+
/**
* @param {any} ctx
* @param {{ toolDefs: any[], defineTool: Function, keyOf: (exec: any) => Promise<string|undefined>, profileRoot: string, downloadRoot?: string, downloadRoots?: (botId: string) => Promise<string[]>, uploadRoots?: (botId: string) => Promise<string[]>, headless: boolean, playwrightUrl?: string, executablePath?: string, actionTimeout?: number, authorize?: (exec: any, botId: string, action: any) => Promise<void>, completeFor?: (exec: any) => Function, connect?: (botId: string, downloadRoots?: string[]) => Promise<string>, getHold?: (botId: string) => any, onHold?: (botId: string, hold: any) => void, subscribeHolds?: (listener: (botId: string, hold: any) => void) => (() => void) | undefined, openSession?: typeof openBotBrowser, api?: object }} options
*/
export function mountBrowserPlugin(ctx, { toolDefs, defineTool, keyOf, profileRoot, downloadRoot, downloadRoots, uploadRoots, headless, playwrightUrl, executablePath, actionTimeout, authorize, completeFor, connect, getHold, onHold, subscribeHolds, openSession = openBotBrowser, api }) {
const sessions = new Map()
++ const names = new Set(toolDefs.map(tool => tool.name))
++ // Tool results persist their rendered blocks, so changing render() cannot fix
++ // old screenshots. Replace only this plugin's malformed image blocks through
++ // DSH's public surface API, keeping the original log for the transcript.
++ ctx.on('agent/pre-step', async ({ agent }, next) => {
++ const session = agent.session
++ let changed = false
++ for (const seq of session.surface?.nodes ?? []) {
++ const event = session.eventAt(seq)
++ if (event.type !== 'tool/result') continue
++ if (!event.sourceEventSeqs?.some(source => names.has(session.eventAt(source)?.data?.name))) continue
++ const message = session.deriveEventMessage(event)
++ if (!message.content.some(block => block.type === 'image' && !block.attachment && typeof block.data === 'string')) continue
++ if (!await keyOf({ agent })) continue
++ const content = []
++ for (const block of message.content) {
++ if (block.type !== 'image' || block.attachment || typeof block.data !== 'string') { content.push(block); continue }
++ const saved = await saveShot(ctx, { text: '', image: block.data })
++ content.push(saved.attachment
++ ? { type: 'image', attachment: saved.attachment, ...(block.offloaded ? { offloaded: true } : {}) }
++ : { type: 'text', text: saved.text.trim() })
++ }
++ session.append('tool/result', { ...event.data, message: { ...message, content } }, {
++ surfaceOp: { op: 'replace', startSeq: seq, endSeq: seq }, sourceEventSeqs: [seq],
++ })
++ changed = true
++ }
++ if (changed) await ctx.sessions.flush(session)
++ return next()
++ }, { prepend: true })
const unsubscribeHolds = subscribeHolds?.((botId, hold) => {
if (hold?.by !== 'user') return
const session = sessions.get(botId)
diff --cc ds-bot/test/browser-plugin-output.test.js
index 052c5cd2,f5d0f2b0..1f6c6966
--- a/ds-bot/test/browser-plugin-output.test.js
+++ b/ds-bot/test/browser-plugin-output.test.js
@@@ -1,7 -1,7 +1,8 @@@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { Context } from '@deepseek-ai/cordis'
--import { ToolCallId } from '@deepseek-ai/dsh-llm'
++import { createToolResultMessage, ToolCallId } from '@deepseek-ai/dsh-llm'
++import { Session } from '@deepseek-ai/dsh-session'
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
import ToolRuntime, { defineTool } from '@deepseek-ai/dsh-tools'
import { mountBrowserPlugin } from '../src/host/browser/mount.js'
@@@ -73,53 -70,13 +74,90 @@@ test('browser handoff and resume retur
const screenshot = await execute('browser_screenshot', {})
assert.equal(screenshot.isError, false, screenshot.error?.message)
- assert.deepEqual(screenshot.value, { text: 'Screenshot of https://example.com/.', image: 'AAAA' })
+ assert.deepEqual(saved, [{
+ mediaType: 'image/jpeg',
+ data: Buffer.from('AAAA', 'base64'),
+ name: 'Browser screenshot',
+ }])
+ assert.deepEqual(screenshot.value, { text: 'Screenshot of https://example.com/.', attachment: ref })
assert.deepEqual(screenshot.content, [
{ type: 'text', text: 'Screenshot of https://example.com/.' },
- { type: 'image', data: 'AAAA', mediaType: 'image/jpeg' },
+ { type: 'image', attachment: ref },
+ ])
+})
+
+test('a screenshot degrades to text when no attachments service is mounted', async t => {
+ const ctx = new Context()
+ await ctx.plugin(SystemPrompt)
+ await ctx.plugin(ToolRuntime)
+ t.after(async () => ctx.fiber.dispose())
+
+ const plugin = inner => {
+ mountBrowserPlugin(inner, {
+ toolDefs: refsTools.filter(tool => tool.name === 'browser_screenshot'),
+ defineTool,
+ keyOf: async () => 'bot-1',
+ profileRoot: '/tmp/browser-profiles',
+ headless: true,
+ openSession: async () => ({
+ async shot() { return { text: 'Screenshot of https://example.com/.', image: 'AAAA' } },
+ async close() {},
+ }),
+ })
+ }
+ plugin.inject = ['tools']
+ await ctx.plugin(plugin)
+
+ const screenshot = await ctx.tools.execute({
+ signal: new AbortController().signal,
+ callId: ToolCallId('browser_screenshot'),
+ name: 'browser_screenshot',
+ arguments: {},
+ })
+ assert.equal(screenshot.isError, false, screenshot.error?.message)
+ assert.deepEqual(screenshot.value, { text: 'Screenshot of https://example.com/. (The screenshot could not be attached.)' })
+ assert.deepEqual(screenshot.content, [
+ { type: 'text', text: 'Screenshot of https://example.com/. (The screenshot could not be attached.)' },
+ ])
+})
+
++test('a stored browser screenshot is repaired before the next step and survives reload', async t => {
++ const ctx = new Context()
++ await ctx.plugin(SystemPrompt)
++ await ctx.plugin(ToolRuntime)
++ t.after(async () => ctx.fiber.dispose())
++ const session = Session.create('old-browser-chat')
++ const call = session.append('tool/call', { turn: 1, step: 1, callId: 'shot', name: 'browser_screenshot', arguments: '{}' })
++ session.append('tool/result', {
++ turn: 1, step: 1,
++ message: createToolResultMessage({
++ callId: 'shot', isError: false,
++ content: [{ type: 'text', text: 'Old screenshot' }, { type: 'image', data: 'AAAA', mediaType: 'image/jpeg', offloaded: true }],
++ }),
++ }, { surfaceOp: 'append', sourceEventSeqs: [call.seq] })
++ const ref = { attachmentId: 'sha256:abcd', mediaType: 'image/jpeg', bytes: 3, width: 1, height: 1 }
++ let flushed
++ const plugin = inner => {
++ inner.provide('attachments', { saveImage: async () => ref })
++ inner.provide('sessions', { flush: async value => { flushed = value.snapshotEvents() } })
++ mountBrowserPlugin(inner, {
++ toolDefs: refsTools, defineTool, keyOf: async () => 'bot-1', profileRoot: '/tmp/browser-profiles', headless: true,
++ })
++ }
++ plugin.inject = ['tools']
++ await ctx.plugin(plugin)
++ const step = () => ctx.waterfall('agent/pre-step', { agent: { id: 'bot-1', session } }, async () => 'next')
++ await step()
++ assert.ok(flushed, 'the repair is durable before the model sees it')
++ const restored = Session.create('old-browser-chat', flushed)
++ assert.deepEqual(restored.deriveMessages()[0].content, [
++ { type: 'text', text: 'Old screenshot' }, { type: 'image', attachment: ref, offloaded: true },
+ ])
++ const afterRepair = session.seq
++ await step()
++ assert.equal(session.seq, afterRepair, 'a repaired result is not rewritten again')
+ })
+
test('browser tools refuse every operation while a user hold is active', async t => {
const ctx = new Context()
await ctx.plugin(SystemPrompt)
diff --cc ds-bot/test/browser-plugins.test.js
index 0361f4d1,0361f4d1..4db92e04
--- a/ds-bot/test/browser-plugins.test.js
+++ b/ds-bot/test/browser-plugins.test.js
@@@ -111,6 -111,6 +111,7 @@@ test('app browser endpoint carries down
}
try {
mountBrowserPlugin({
++ on() {},
effect(create) { cleanup = create() },
tools: { register(tool) { tools.set(tool.name, tool) } },
}, {
@@@ -160,6 -160,6 +161,7 @@@ test('concurrent first browser calls sh
let connectCalls = 0
let rootsCalls = 0
mountBrowserPlugin({
++ on() {},
effect(create) { cleanup = create() },
tools: { register(tool) { tools.set(tool.name, tool) } },
}, {
@@@ -211,6 -211,6 +213,7 @@@ test('user takeover while opening close
const closed = new Promise(resolve => { resolveClosed = resolve })
let hold = null
mountBrowserPlugin({
++ on() {},
effect(create) { cleanup = create() },
tools: { register(tool) { tools.set(tool.name, tool) } },
}, {
@@@ -246,6 -246,6 +249,7 @@@ test('user takeover refuses a browser c
let openCalls = 0
let connectCalls = 0
mountBrowserPlugin({
++ on() {},
effect() {},
tools: { register(tool) { tools.set(tool.name, tool) } },
}, {