A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Repair persisted browser screenshots before the next model step
Fei24114b8138215abc
diff --cc ds-bot/src/host/browser/mount.jsindex 31afe0a4,f4b95cc6..0fc58b21--- a/ds-bot/src/host/browser/mount.js+++ b/ds-bot/src/host/browser/mount.js@@@ -43,46 -29,12 +43,62 @@@ export const browserOutput =},}- function attachmentsOf(ctx) {- try {- const service = ctx.get?.('attachments')- if (service !== undefined) return service- } catch {- // Not a cordis context, or the service was never provided.- }- try {- return ctx.attachments- } catch {- return undefined- }- }-+// Tool bodies return screenshot bytes; the model-facing result must carry the+// stored attachment instead, since content blocks cannot inline image data.+async function saveShot(ctx, value) {+ if (!value || typeof value !== 'object' || typeof value.image !== 'string') return value- const attachments = attachmentsOf(ctx)++ const attachments = ctx.get('attachments')+ if (typeof attachments?.saveImage === 'function') {+ try {+ const attachment = await attachments.saveImage({+ mediaType: 'image/jpeg',+ data: Buffer.from(value.image, 'base64'),+ name: 'Browser screenshot',+ })+ return { text: value.text, attachment }+ } catch (error) {+ ctx.logger?.warn?.('ds-bot: screenshot could not be stored: %s', error?.message ?? error)+ }+ }+ return { text: `${value.text ?? ''} (The screenshot could not be attached.)` }+}+/*** @param {any} ctx* @param {{ toolDefs: any[], defineTool: Function, keyOf: (exec: any) => Promise<string|undefined>, profileRoot: string, downloadRoot?: string, downloadRoots?: (botId: string) => Promise<string[]>, uploadRoots?: (botId: string) => Promise<string[]>, headless: boolean, playwrightUrl?: string, executablePath?: string, actionTimeout?: number, authorize?: (exec: any, botId: string, action: any) => Promise<void>, completeFor?: (exec: any) => Function, connect?: (botId: string, downloadRoots?: string[]) => Promise<string>, getHold?: (botId: string) => any, onHold?: (botId: string, hold: any) => void, subscribeHolds?: (listener: (botId: string, hold: any) => void) => (() => void) | undefined, openSession?: typeof openBotBrowser, api?: object }} options*/export function mountBrowserPlugin(ctx, { toolDefs, defineTool, keyOf, profileRoot, downloadRoot, downloadRoots, uploadRoots, headless, playwrightUrl, executablePath, actionTimeout, authorize, completeFor, connect, getHold, onHold, subscribeHolds, openSession = openBotBrowser, api }) {const sessions = new Map()++ const names = new Set(toolDefs.map(tool => tool.name))++ // Tool results persist their rendered blocks, so changing render() cannot fix++ // old screenshots. Replace only this plugin's malformed image blocks through++ // DSH's public surface API, keeping the original log for the transcript.++ ctx.on('agent/pre-step', async ({ agent }, next) => {++ const session = agent.session++ let changed = false++ for (const seq of session.surface?.nodes ?? []) {++ const event = session.eventAt(seq)++ if (event.type !== 'tool/result') continue++ if (!event.sourceEventSeqs?.some(source => names.has(session.eventAt(source)?.data?.name))) continue++ const message = session.deriveEventMessage(event)++ if (!message.content.some(block => block.type === 'image' && !block.attachment && typeof block.data === 'string')) continue++ if (!await keyOf({ agent })) continue++ const content = []++ for (const block of message.content) {++ if (block.type !== 'image' || block.attachment || typeof block.data !== 'string') { content.push(block); continue }++ const saved = await saveShot(ctx, { text: '', image: block.data })++ content.push(saved.attachment++ ? { type: 'image', attachment: saved.attachment, ...(block.offloaded ? { offloaded: true } : {}) }++ : { type: 'text', text: saved.text.trim() })++ }++ session.append('tool/result', { ...event.data, message: { ...message, content } }, {++ surfaceOp: { op: 'replace', startSeq: seq, endSeq: seq }, sourceEventSeqs: [seq],++ })++ changed = true++ }++ if (changed) await ctx.sessions.flush(session)++ return next()++ }, { prepend: true })const unsubscribeHolds = subscribeHolds?.((botId, hold) => {if (hold?.by !== 'user') returnconst session = sessions.get(botId)diff --cc ds-bot/test/browser-plugin-output.test.jsindex 052c5cd2,f5d0f2b0..1f6c6966--- a/ds-bot/test/browser-plugin-output.test.js+++ b/ds-bot/test/browser-plugin-output.test.js@@@ -1,7 -1,7 +1,8 @@@import assert from 'node:assert/strict'import { test } from 'node:test'import { Context } from '@deepseek-ai/cordis'--import { ToolCallId } from '@deepseek-ai/dsh-llm'++import { createToolResultMessage, ToolCallId } from '@deepseek-ai/dsh-llm'++import { Session } from '@deepseek-ai/dsh-session'import SystemPrompt from '@deepseek-ai/dsh-system-prompt'import ToolRuntime, { defineTool } from '@deepseek-ai/dsh-tools'import { mountBrowserPlugin } from '../src/host/browser/mount.js'@@@ -73,53 -70,13 +74,90 @@@ test('browser handoff and resume returconst screenshot = await execute('browser_screenshot', {})assert.equal(screenshot.isError, false, screenshot.error?.message)- assert.deepEqual(screenshot.value, { text: 'Screenshot of https://example.com/.', image: 'AAAA' })+ assert.deepEqual(saved, [{+ mediaType: 'image/jpeg',+ data: Buffer.from('AAAA', 'base64'),+ name: 'Browser screenshot',+ }])+ assert.deepEqual(screenshot.value, { text: 'Screenshot of https://example.com/.', attachment: ref })assert.deepEqual(screenshot.content, [{ type: 'text', text: 'Screenshot of https://example.com/.' },- { type: 'image', data: 'AAAA', mediaType: 'image/jpeg' },+ { type: 'image', attachment: ref },+ ])+})++test('a screenshot degrades to text when no attachments service is mounted', async t => {+ const ctx = new Context()+ await ctx.plugin(SystemPrompt)+ await ctx.plugin(ToolRuntime)+ t.after(async () => ctx.fiber.dispose())++ const plugin = inner => {+ mountBrowserPlugin(inner, {+ toolDefs: refsTools.filter(tool => tool.name === 'browser_screenshot'),+ defineTool,+ keyOf: async () => 'bot-1',+ profileRoot: '/tmp/browser-profiles',+ headless: true,+ openSession: async () => ({+ async shot() { return { text: 'Screenshot of https://example.com/.', image: 'AAAA' } },+ async close() {},+ }),+ })+ }+ plugin.inject = ['tools']+ await ctx.plugin(plugin)++ const screenshot = await ctx.tools.execute({+ signal: new AbortController().signal,+ callId: ToolCallId('browser_screenshot'),+ name: 'browser_screenshot',+ arguments: {},+ })+ assert.equal(screenshot.isError, false, screenshot.error?.message)+ assert.deepEqual(screenshot.value, { text: 'Screenshot of https://example.com/. (The screenshot could not be attached.)' })+ assert.deepEqual(screenshot.content, [+ { type: 'text', text: 'Screenshot of https://example.com/. (The screenshot could not be attached.)' },+ ])+})+++test('a stored browser screenshot is repaired before the next step and survives reload', async t => {++ const ctx = new Context()++ await ctx.plugin(SystemPrompt)++ await ctx.plugin(ToolRuntime)++ t.after(async () => ctx.fiber.dispose())++ const session = Session.create('old-browser-chat')++ const call = session.append('tool/call', { turn: 1, step: 1, callId: 'shot', name: 'browser_screenshot', arguments: '{}' })++ session.append('tool/result', {++ turn: 1, step: 1,++ message: createToolResultMessage({++ callId: 'shot', isError: false,++ content: [{ type: 'text', text: 'Old screenshot' }, { type: 'image', data: 'AAAA', mediaType: 'image/jpeg', offloaded: true }],++ }),++ }, { surfaceOp: 'append', sourceEventSeqs: [call.seq] })++ const ref = { attachmentId: 'sha256:abcd', mediaType: 'image/jpeg', bytes: 3, width: 1, height: 1 }++ let flushed++ const plugin = inner => {++ inner.provide('attachments', { saveImage: async () => ref })++ inner.provide('sessions', { flush: async value => { flushed = value.snapshotEvents() } })++ mountBrowserPlugin(inner, {++ toolDefs: refsTools, defineTool, keyOf: async () => 'bot-1', profileRoot: '/tmp/browser-profiles', headless: true,++ })++ }++ plugin.inject = ['tools']++ await ctx.plugin(plugin)++ const step = () => ctx.waterfall('agent/pre-step', { agent: { id: 'bot-1', session } }, async () => 'next')++ await step()++ assert.ok(flushed, 'the repair is durable before the model sees it')++ const restored = Session.create('old-browser-chat', flushed)++ assert.deepEqual(restored.deriveMessages()[0].content, [++ { type: 'text', text: 'Old screenshot' }, { type: 'image', attachment: ref, offloaded: true },+ ])++ const afterRepair = session.seq++ await step()++ assert.equal(session.seq, afterRepair, 'a repaired result is not rewritten again')+ })+test('browser tools refuse every operation while a user hold is active', async t => {const ctx = new Context()await ctx.plugin(SystemPrompt)diff --cc ds-bot/test/browser-plugins.test.jsindex 0361f4d1,0361f4d1..4db92e04--- a/ds-bot/test/browser-plugins.test.js+++ b/ds-bot/test/browser-plugins.test.js@@@ -111,6 -111,6 +111,7 @@@ test('app browser endpoint carries down}try {mountBrowserPlugin({++ on() {},effect(create) { cleanup = create() },tools: { register(tool) { tools.set(tool.name, tool) } },}, {@@@ -160,6 -160,6 +161,7 @@@ test('concurrent first browser calls shlet connectCalls = 0let rootsCalls = 0mountBrowserPlugin({++ on() {},effect(create) { cleanup = create() },tools: { register(tool) { tools.set(tool.name, tool) } },}, {@@@ -211,6 -211,6 +213,7 @@@ test('user takeover while opening closeconst closed = new Promise(resolve => { resolveClosed = resolve })let hold = nullmountBrowserPlugin({++ on() {},effect(create) { cleanup = create() },tools: { register(tool) { tools.set(tool.name, tool) } },}, {@@@ -246,6 -246,6 +249,7 @@@ test('user takeover refuses a browser clet openCalls = 0let connectCalls = 0mountBrowserPlugin({++ on() {},effect() {},tools: { register(tool) { tools.set(tool.name, tool) } },}, {