A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Avoid sending screenshots to text-only browser routes
Fei2411a0c2c3d0baf3
diff --git a/ds-bot/src/host/browser/session-model.js b/ds-bot/src/host/browser/session-model.jsindex 7cadf7dc..b51e74d8 100644--- a/ds-bot/src/host/browser/session-model.js+++ b/ds-bot/src/host/browser/session-model.js@@ -45,7 +45,7 @@ function attachmentsOf(ctx) {}}-async function userContent(content, attachments) {+async function userContent(content, attachments, readsImages) {const parts = Array.isArray(content) ? content : [{ type: 'text', text: String(content ?? '') }]const blocks = []const pending = []@@ -58,6 +58,11 @@ async function userContent(content, attachments) {}if (part?.type === 'image_url') {sawImage = true+ if (!readsImages) {+ blocks.push({ type: 'text', text: OMITTED_IMAGE })+ rejectedImage = true+ continue+ }const input = imageInput(part)if (!input || typeof attachments?.saveImages !== 'function') {blocks.push({ type: 'text', text: OMITTED_IMAGE })@@ -108,7 +113,7 @@ function assistantContent(message, config) {})}-async function dshMessages(messages, attachments, config) {+async function dshMessages(messages, attachments, config, readsImages) {const output = []let sawImage = falselet rejectedImage = false@@ -116,7 +121,7 @@ async function dshMessages(messages, attachments, config) {if (message.role === 'system') {output.push(createSystemMessage(textParts(message.content).join('\n')))} else if (message.role === 'user') {- const content = await userContent(message.content, attachments)+ const content = await userContent(message.content, attachments, readsImages)output.push(createUserMessage({ content: content.blocks, source: { kind: 'user' } }))sawImage ||= content.sawImagerejectedImage ||= content.rejectedImage@@ -158,7 +163,11 @@ async function routeReadsImages(ctx, config, rt) {export function sessionComplete(ctx, exec, rt) {return async ({ messages, tools }) => {const { header, sessionId } = routeOf(exec)- const converted = await dshMessages(messages, attachmentsOf(ctx), header.config)+ const hasImages = (messages ?? []).some(message => (+ Array.isArray(message.content) && message.content.some(part => part?.type === 'image_url')+ ))+ const readsImages = hasImages ? await routeReadsImages(ctx, header.config, rt) : true+ const converted = await dshMessages(messages, attachmentsOf(ctx), header.config, readsImages)const requestMessages = typeof rt?.tagUsage === 'function'? rt.tagUsage(converted.messages, { sessionId, kind: 'browser-task' }): converted.messages@@ -186,7 +195,7 @@ export function sessionComplete(ctx, exec, rt) {if (reasoning) message.reasoning_content = reasoningif (toolCalls.length > 0) message.tool_calls = toolCallsconst vision = converted.sawImage- ? converted.rejectedImage || !await routeReadsImages(ctx, header.config, rt) ? 'rejected' : 'accepted'+ ? converted.rejectedImage || !readsImages ? 'rejected' : 'accepted': undefinedreturn {choices: [{ message, finish_reason: toolCalls.length > 0 ? 'tool_calls' : finish.kind === 'max-tokens' ? 'length' : 'stop' }],diff --git a/ds-bot/test/session-model.test.js b/ds-bot/test/session-model.test.jsindex a40a2a82..f8418354 100644--- a/ds-bot/test/session-model.test.js+++ b/ds-bot/test/session-model.test.js@@ -7,9 +7,10 @@ const header = {tools: [{ name: 'session-private-tool', description: 'not for browser_task', parameters: {} }],}-function fixture(chunks, { attachments, inputModalities } = {}) {+function fixture(chunks, { attachments, inputModalities, textOnlyModels } = {}) {const requests = []const tags = []+ let modelInfoCalls = 0const ctx = {get: name => name === 'attachments' ? attachments : undefined,llm: {@@ -18,6 +19,7 @@ function fixture(chunks, { attachments, inputModalities } = {}) {yield* chunks},async resolveModelInfo() {+ modelInfoCalls += 1return { inputModalities }},},@@ -27,12 +29,13 @@ function fixture(chunks, { attachments, inputModalities } = {}) {requestHeader: () => header,}const rt = {+ config: { textOnlyModels },tagUsage(messages, options) {tags.push(options)return messages},}- return { ctx, exec: { agent: { session }, signal: new AbortController().signal }, rt, requests, tags }+ return { ctx, exec: { agent: { session }, signal: new AbortController().signal }, rt, requests, tags, modelInfoCalls: () => modelInfoCalls }}const stop = { type: 'finish', reason: { kind: 'stop' } }@@ -50,6 +53,7 @@ test('session completion converts DSH text to an OpenAI chat response', async ()assert.equal(result.choices[0].message.content, 'The page title is Example.')assert.deepEqual(setup.tags, [{ sessionId: 'session-browser-task', kind: 'browser-task' }])+ assert.equal(setup.modelInfoCalls(), 0)})test('session completion preserves DSH tool-call identity, name, and JSON arguments', async () => {@@ -118,7 +122,55 @@ test('follow-up assistant tool call and tool result are converted to DSH messageassert.deepEqual(setup.tags, [{ sessionId: 'session-browser-task', kind: 'browser-task' }])})-test('browser screenshots become DSH image blocks when the attachment store is available', async () => {+test('text-only model metadata omits screenshots before saving attachments', async () => {+ let saveCalls = 0+ const attachments = { async saveImages() { saveCalls += 1; return [] } }+ const setup = fixture([stop], { attachments, inputModalities: ['text'] })+ const result = await sessionComplete(setup.ctx, setup.exec, setup.rt)({+ messages: [{+ role: 'user',+ content: [+ { type: 'text', text: 'Screenshot.' },+ { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,/9j/4AA=' } },+ ],+ }],+ tools: [],+ })++ assert.deepEqual(setup.requests[0].messages[0].content, [+ { type: 'text', text: 'Screenshot.' },+ { type: 'text', text: '[Browser screenshot omitted.]' },+ ])+ assert.equal(setup.requests[0].messages[0].content.some(part => part.type === 'image'), false)+ assert.equal(saveCalls, 0)+ assert.equal(result.vision, 'rejected')+ assert.equal(setup.modelInfoCalls(), 1)+})++test('configured text-only models omit screenshots without attachment saves', async () => {+ let saveCalls = 0+ const attachments = { async saveImages() { saveCalls += 1; return [] } }+ const setup = fixture([stop], {+ attachments,+ inputModalities: ['text', 'image'],+ textOnlyModels: ['mock-provider/session-model'],+ })+ const result = await sessionComplete(setup.ctx, setup.exec, setup.rt)({+ messages: [{+ role: 'user',+ content: [{ type: 'image_url', image_url: { url: 'data:image/jpeg;base64,/9j/4AA=' } }],+ }],+ tools: [],+ })++ assert.equal(setup.requests[0].messages[0].content.some(part => part.type === 'image'), false)+ assert.equal(setup.requests[0].messages[0].content[0].text, '[Browser screenshot omitted.]')+ assert.equal(saveCalls, 0)+ assert.equal(result.vision, 'rejected')+ assert.equal(setup.modelInfoCalls(), 0)+})++test('browser screenshots become DSH image blocks when the route supports images', async () => {const saved = []const attachments = {async saveImages(inputs) {@@ -133,7 +185,7 @@ test('browser screenshots become DSH image blocks when the attachment store is a},}const setup = fixture([stop], { attachments, inputModalities: ['text', 'image'] })- await sessionComplete(setup.ctx, setup.exec)({+ const result = await sessionComplete(setup.ctx, setup.exec, setup.rt)({messages: [{role: 'user',content: [@@ -156,6 +208,7 @@ test('browser screenshots become DSH image blocks when the attachment store is aheight: 1,},})+ assert.equal(result.vision, 'accepted')})test('missing Bot session model fails with a clear error', async () => {