Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

Route browser_task through the Bot session model

Fei2411305ec58ff617
diff --git a/browser-plugins/README.md b/browser-plugins/README.md
index 6a95873e..7f217b2f 100644
--- a/browser-plugins/README.md
+++ b/browser-plugins/README.md
@@ -20,6 +20,10 @@ node bench/run.mjs
node bench/render-report.mjs
```
-模型走 OpenDesign `https://amr-link.open-design.ai/v1`,模型 id 是 `deepseek-v4.1-flash`。密钥用环境变量 `OPENDESIGN_API_KEY`。不要写进仓库。
+`browser_task` 使用调用它的 Bot 在 DSH 中配置的模型。跑基准测试前设置 `BENCH_BASE_URL`、`BENCH_MODEL` 和 `BENCH_API_KEY`。
+
+```sh
+BENCH_BASE_URL=... BENCH_MODEL=... BENCH_API_KEY=... node bench/run.mjs
+```
设计说明在 `docs/plans/2026-10-21-browser-plugins.md`。
diff --git a/browser-plugins/bench/deepseek.js b/browser-plugins/bench/deepseek.js
new file mode 100644
index 00000000..6b8f3e51
--- /dev/null
+++ b/browser-plugins/bench/deepseek.js
@@ -0,0 +1,55 @@
+const ENDPOINT_PATH = '/chat/completions'
+
+export function benchConfig() {
+ return {
+ baseUrl: process.env.BENCH_BASE_URL?.replace(/\/$/, ''),
+ model: process.env.BENCH_MODEL,
+ apiKey: process.env.BENCH_API_KEY,
+ }
+}
+
+export async function completeBench({ messages, tools, timeoutMs = 90_000 }) {
+ const { baseUrl, model, apiKey } = benchConfig()
+ if (!baseUrl || !model || !apiKey) {
+ throw new Error('Set BENCH_BASE_URL, BENCH_MODEL, and BENCH_API_KEY to run the browser benchmark.')
+ }
+ const send = async (payload, signal) => {
+ const response = await fetch(`${baseUrl}${ENDPOINT_PATH}`, {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', authorization: `Bearer ${apiKey}` },
+ body: JSON.stringify(payload),
+ signal,
+ })
+ const body = await response.json().catch(() => ({}))
+ if (!response.ok) {
+ const detail = body?.error?.message || response.statusText
+ throw new Error(`Benchmark model ${response.status}: ${detail}`)
+ }
+ return body
+ }
+ const payload = { model, messages, max_tokens: 4096 }
+ if (tools?.length) payload.tools = tools
+ const controller = new AbortController()
+ const timer = setTimeout(() => controller.abort(), timeoutMs)
+ try {
+ const body = await send(payload, controller.signal)
+ return { ...body, vision: 'accepted' }
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error)
+ if (!/image|multimodal|vision|unsupported/i.test(message)) throw error
+ const retry = new AbortController()
+ const retryTimer = setTimeout(() => retry.abort(), timeoutMs)
+ try {
+ const stripped = messages.map(message => {
+ if (!Array.isArray(message.content)) return message
+ const text = message.content.filter(part => part.type === 'text').map(part => part.text).join('\n')
+ return { ...message, content: text || '(screenshot omitted)' }
+ })
+ return { ...(await send({ ...payload, messages: stripped }, retry.signal)), vision: 'rejected' }
+ } finally {
+ clearTimeout(retryTimer)
+ }
+ } finally {
+ clearTimeout(timer)
+ }
+}
diff --git a/browser-plugins/bench/render-report.mjs b/browser-plugins/bench/render-report.mjs
index b4e5b77b..94f935ad 100644
--- a/browser-plugins/bench/render-report.mjs
+++ b/browser-plugins/bench/render-report.mjs
@@ -10,7 +10,7 @@ const esc = (value) => String(value ?? '')
.replace(/>/g, '>')
.replace(/"/g, '"')
-let results = { key: 'missing', driver: [], isolation: {}, plugins: {}, model: 'deepseek-v4.1-flash' }
+let results = { key: 'missing', driver: [], isolation: {}, plugins: {}, model: process.env.BENCH_MODEL ?? '' }
try {
results = JSON.parse(await readFile(new URL('./results.json', outDir), 'utf8'))
} catch { /* rendered before the bench */ }
@@ -161,7 +161,7 @@ const html = `<!doctype html>
<section id="runs">
<h2>实测</h2>
<p>标准用了公开的 <a href="https://github.com/MinorJerry/WebVoyager">WebVoyager</a>(browser-use 也用这套报过成绩)。整套六百多条、很多站要登录或已经过期,不全跑。抽了两条 arXiv 原文任务:<code>ArXiv--11</code>、<code>ArXiv--6</code>。另外用自建页面补 WebVoyager 不覆盖的部分:检索后再点进去、翻页、Shadow DOM、同源 iframe、只存在于 canvas 的字、登录必须交给人。</p>
- <p>模型 <code>${esc(results.model || 'deepseek-v4.1-flash')}</code>,经 OpenDesign <code>https://amr-link.open-design.ai/v1</code>。密钥:${results.key === 'present' ? '用的是环境里的 OPENDESIGN_API_KEY,没有写入报告。' : '这台环境里没有 OPENDESIGN_API_KEY,模型任务没有跑。'}</p>
+ <p>模型 <code>${esc(results.model || 'BENCH_MODEL')}</code>,地址由 <code>BENCH_BASE_URL</code> 配置。${results.key === 'present' ? '使用了环境变量 BENCH_API_KEY,没有写入报告。' : '模型任务需要配置 BENCH_BASE_URL、BENCH_MODEL 和 BENCH_API_KEY。'}</p>
<h3>驱动,不经过模型</h3>
<ul>${driverRows || '<li>还没有驱动结果</li>'}</ul>
<h3>两个 Bot 的 cookie</h3>
diff --git a/browser-plugins/bench/run.mjs b/browser-plugins/bench/run.mjs
index 296a11f1..1c873c74 100644
--- a/browser-plugins/bench/run.mjs
+++ b/browser-plugins/bench/run.mjs
@@ -1,7 +1,7 @@
import { mkdir, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
-import { completeDeepseek, deepseekKey, MODEL } from '../../ds-bot/src/host/browser/deepseek.js'
+import { benchConfig, completeBench } from './deepseek.js'
import { runToolLoop } from '../../ds-bot/src/host/browser/loop.js'
import * as refs from '../../ds-bot/src/host/browser/refs.js'
import * as script from '../../ds-bot/src/host/browser/script.js'
@@ -89,14 +89,14 @@ function bind(plugin, session) {
name: tool.name,
description: tool.description,
parameters: tool.parameters,
- execute: (args) => tool.execute(args, { session, botId: session.botId, profileDir: session.userDataDir, complete: completeDeepseek }),
+ execute: (args) => tool.execute(args, { session, botId: session.botId, profileDir: session.userDataDir, complete: completeBench }),
}))
}
async function judge(task, answer, pageText) {
if (task.check) return { ok: false, detail: 'local tasks are judged in code' }
try {
- const response = await completeDeepseek({
+ const response = await completeBench({
messages: [
{ role: 'system', content: 'Judge a web agent. Reply with JSON only: {"success":true|false,"reason":"..."}.' },
{ role: 'user', content: `Task: ${task.goal('')}\nAgent answer:\n${answer || '(empty)'}\nFinal page text:\n${pageText.slice(0, 4000)}` },
@@ -115,10 +115,10 @@ async function runModelTask(plugin, task, session, fixture) {
const started = Date.now()
try {
const result = plugin.id === 'task'
- ? await runTask({ session, goal, complete: completeDeepseek, maxSteps: task.maxSteps })
+ ? await runTask({ session, goal, complete: completeBench, maxSteps: task.maxSteps })
: await runToolLoop({
tools: bind(plugin, session),
- complete: completeDeepseek,
+ complete: completeBench,
goal,
system: plugin.systemPrompt,
maxSteps: task.maxSteps,
@@ -188,18 +188,18 @@ const driverSession = await openBotBrowser({ botId: 'driver', profileRoot, headl
const driver = await drive(driverSession, fixture)
await driverSession.close()
-const key = deepseekKey()
+const { baseUrl, model, apiKey } = benchConfig()
const results = {
- model: MODEL,
- endpoint: 'https://amr-link.open-design.ai/v1',
- key: key ? 'present' : 'missing',
+ model: model ?? '',
+ endpoint: baseUrl ?? '',
+ key: apiKey ? 'present' : 'missing',
driver,
isolation: await isolate(playwrightUrl),
plugins: {},
}
-if (!key) {
- for (const name of Object.keys(selectedPlugins)) results.plugins[name] = { skipped: 'OPENDESIGN_API_KEY is not set in this environment' }
+if (!baseUrl || !model || !apiKey) {
+ for (const name of Object.keys(selectedPlugins)) results.plugins[name] = { skipped: 'Set BENCH_BASE_URL, BENCH_MODEL, and BENCH_API_KEY to run model tasks.' }
} else {
for (const [name, plugin] of Object.entries(selectedPlugins)) {
const rows = []
diff --git a/browser-plugins/bench/upstream_browser_use.py b/browser-plugins/bench/upstream_browser_use.py
index d7bf8335..74ef2b37 100644
--- a/browser-plugins/bench/upstream_browser_use.py
+++ b/browser-plugins/bench/upstream_browser_use.py
@@ -70,9 +70,9 @@ async def one(llm, task_id, prompt, check):
async def main():
llm = ChatOpenAI(
- model='deepseek-v4.1-flash',
- base_url='https://amr-link.open-design.ai/v1',
- api_key=os.environ['OPENDESIGN_API_KEY'],
+ model=os.environ['BENCH_MODEL'],
+ base_url=os.environ['BENCH_BASE_URL'],
+ api_key=os.environ['BENCH_API_KEY'],
dont_force_structured_output=True,
timeout=90,
)
diff --git a/browser-plugins/bench/upstream_tools.mjs b/browser-plugins/bench/upstream_tools.mjs
index f7ee7a63..9f690ecf 100644
--- a/browser-plugins/bench/upstream_tools.mjs
+++ b/browser-plugins/bench/upstream_tools.mjs
@@ -1,8 +1,8 @@
-// Drive two real browser stacks with DeepSeek V4.1 Flash:
+// Drive two real browser stacks with the configured benchmark model:
// Microsoft @playwright/mcp, and the published npm package dsh-plugin-browser-use.
import { spawn } from 'node:child_process'
import { readFile } from 'node:fs/promises'
-import { completeDeepseek } from '../../ds-bot/src/host/browser/deepseek.js'
+import { completeBench, benchConfig } from './deepseek.js'
const ORIGIN = process.env.FIXTURE_ORIGIN
const CHROME = process.env.CHROME_PATH
@@ -31,7 +31,7 @@ async function runLoop({ tools, call, goal, maxSteps = 8 }) {
]
const trace = []
for (let step = 0; step < maxSteps; step += 1) {
- const response = await completeDeepseek({ messages, tools })
+ const response = await completeBench({ messages, tools })
const message = response.choices?.[0]?.message ?? {}
const calls = message.tool_calls ?? []
messages.push({
@@ -257,7 +257,8 @@ async function runDsh() {
const { writeFile } = await import('node:fs/promises')
const outPath = new URL('./out/upstream-tools.json', import.meta.url)
-const out = { model: 'deepseek-v4.1-flash', endpoint: 'https://amr-link.open-design.ai/v1' }
+const { model, baseUrl } = benchConfig()
+const out = { model: model ?? '', endpoint: baseUrl ?? '' }
const save = () => writeFile(outPath, JSON.stringify(out, null, 2))
if (which.includes('mcp')) {
try { out.playwrightMcp = await runMcp() } catch (error) { out.playwrightMcpError = String(error).slice(0, 500) }
diff --git a/browser-plugins/refs/index.js b/browser-plugins/refs/index.js
index 57f1fc14..19a358a9 100644
--- a/browser-plugins/refs/index.js
+++ b/browser-plugins/refs/index.js
@@ -1,7 +1,6 @@
import { join } from 'node:path'
import { tmpdir } from 'node:os'
import { defineTool } from '@deepseek-ai/dsh-tools'
-import { completeDeepseek } from '../../ds-bot/src/host/browser/deepseek.js'
import { mountBrowserPlugin } from '../../ds-bot/src/host/browser/mount.js'
import { id, label, systemPrompt, toolDefs } from '../../ds-bot/src/host/browser/refs.js'
@@ -17,7 +16,6 @@ export function apply(ctx, config = {}) {
profileRoot: config.profileRoot ?? join(tmpdir(), 'ds-bot-browser-refs'),
headless: config.headless !== false,
playwrightUrl: config.playwrightUrl,
- complete: completeDeepseek,
async keyOf(exec) {
return config.botId ?? exec.agent?.session?.header?.id ?? exec.agent?.id ?? 'session'
},
diff --git a/browser-plugins/script/index.js b/browser-plugins/script/index.js
index 6652889d..994af35c 100644
--- a/browser-plugins/script/index.js
+++ b/browser-plugins/script/index.js
@@ -1,7 +1,6 @@
import { join } from 'node:path'
import { tmpdir } from 'node:os'
import { defineTool } from '@deepseek-ai/dsh-tools'
-import { completeDeepseek } from '../../ds-bot/src/host/browser/deepseek.js'
import { mountBrowserPlugin } from '../../ds-bot/src/host/browser/mount.js'
import { id, label, systemPrompt, toolDefs } from '../../ds-bot/src/host/browser/script.js'
@@ -15,7 +14,6 @@ export function apply(ctx, config = {}) {
profileRoot: config.profileRoot ?? join(tmpdir(), 'ds-bot-browser-script'),
headless: config.headless !== false,
playwrightUrl: config.playwrightUrl,
- complete: completeDeepseek,
async keyOf(exec) {
return config.botId ?? exec.agent?.session?.header?.id ?? exec.agent?.id ?? 'session'
},
diff --git a/browser-plugins/task/index.js b/browser-plugins/task/index.js
index 72902e19..3b15ff8b 100644
--- a/browser-plugins/task/index.js
+++ b/browser-plugins/task/index.js
@@ -1,8 +1,8 @@
import { join } from 'node:path'
import { tmpdir } from 'node:os'
import { defineTool } from '@deepseek-ai/dsh-tools'
-import { completeDeepseek } from '../../ds-bot/src/host/browser/deepseek.js'
import { mountBrowserPlugin } from '../../ds-bot/src/host/browser/mount.js'
+import { sessionComplete } from '../../ds-bot/src/host/browser/session-model.js'
import { id, label, systemPrompt, toolDefs } from '../../ds-bot/src/host/browser/task.js'
export { id, label, systemPrompt, toolDefs }
@@ -15,7 +15,7 @@ export function apply(ctx, config = {}) {
profileRoot: config.profileRoot ?? join(tmpdir(), 'ds-bot-browser-task'),
headless: config.headless !== false,
playwrightUrl: config.playwrightUrl,
- complete: completeDeepseek,
+ completeFor: exec => sessionComplete(ctx, exec),
async keyOf(exec) {
return config.botId ?? exec.agent?.session?.header?.id ?? exec.agent?.id ?? 'session'
},
diff --git a/browser-plugins/test/write-safety.test.js b/browser-plugins/test/write-safety.test.js
index e4c63c43..b0828989 100644
--- a/browser-plugins/test/write-safety.test.js
+++ b/browser-plugins/test/write-safety.test.js
@@ -387,7 +387,7 @@ test('browser_task driver tools inherit the outer authorization gate', async (t)
headless: true,
playwrightUrl,
actionTimeout: 500,
- complete,
+ completeFor: () => complete,
authorize: async (_exec, _botId, action) => { gateActions.push(action) },
})
const tool = registered.get('browser_task')
diff --git a/docs/plans/2026-10-21-browser-plugins.md b/docs/plans/2026-10-21-browser-plugins.md
index ba74a866..fc074459 100644
--- a/docs/plans/2026-10-21-browser-plugins.md
+++ b/docs/plans/2026-10-21-browser-plugins.md
@@ -47,5 +47,5 @@ Bot 是长期身份。登录应该跟着 Bot 走,而不是跟着某一段对
## 变更
- 2026-10-21:维护者定下全团队一个后端、每 Bot 一份 profile、cookie 不同步。三个插件和实测在同一分支,默认关闭。
-- 2026-10-22:实测模型是 OpenDesign 上的 `deepseek-v4.1-flash`(`https://amr-link.open-design.ai/v1`),密钥用 `OPENDESIGN_API_KEY`。
+- 2026-10-22:`browser_task` 使用调用 Bot 在 DSH 中配置的模型;基准测试模型由 `BENCH_BASE_URL`、`BENCH_MODEL` 和 `BENCH_API_KEY` 配置。
- 2026-10-23:维护者定下权限规则:默认放行;发帖、上传、下载、删除按预授权或审批;删除强制多核对一轮;付款一律交给用户;预授权可设时限,在设置的浏览器页管理。
diff --git a/ds-bot/src/host/browser-plugins.js b/ds-bot/src/host/browser-plugins.js
index a3b1de92..2927504d 100644
--- a/ds-bot/src/host/browser-plugins.js
+++ b/ds-bot/src/host/browser-plugins.js
@@ -2,8 +2,8 @@
// directory, including the group Sessions where that Bot speaks.
import { join } from 'node:path'
import { defineTool } from '@deepseek-ai/dsh-tools'
-import { completeDeepseek } from './browser/deepseek.js'
import { mountBrowserPlugin } from './browser/mount.js'
+import { sessionComplete } from './browser/session-model.js'
import * as refs from './browser/refs.js'
import * as script from './browser/script.js'
import * as task from './browser/task.js'
@@ -13,6 +13,10 @@ export const browserPlugins = { refs, script, task }
const NAMES = Object.keys(browserPlugins)
const GRANT_KINDS = new Set(['post', 'upload', 'download', 'delete'])
+export function defaultBrowserHeadless(platform, env = process.env) {
+ return platform === 'linux' && !env.DISPLAY && !env.WAYLAND_DISPLAY
+}
+
function optionalString(config, key) {
const value = config[key]
if (value !== undefined && typeof value !== 'string') throw new TypeError(`${key} must be a string`)
@@ -88,7 +92,7 @@ export function install(rt) {
const executablePath = optionalString(rt.config, 'browserExecutablePath')
const headless = typeof rt.config.browserHeadless === 'boolean'
? rt.config.browserHeadless
- : !process.env.DISPLAY
+ : defaultBrowserHeadless(process.platform)
const downloadRoot = join(rt.home, 'browser-downloads')
mountBrowserPlugin(rt.ctx, {
toolDefs: plugin.tools ?? plugin.toolDefs,
@@ -105,7 +109,9 @@ export function install(rt) {
playwrightUrl,
executablePath,
authorize: (exec, botId, action) => authorizeBrowserAction(rt, exec, botId, action),
- complete: rt.config.browserComplete ?? completeDeepseek,
+ completeFor: rt.config.browserComplete
+ ? () => rt.config.browserComplete
+ : exec => sessionComplete(rt.ctx, exec, rt),
async keyOf(exec) {
await rt.load()
const id = rt.selfOf(exec.agent?.session?.header?.id ?? exec.agent?.id)
diff --git a/ds-bot/src/host/browser/deepseek.js b/ds-bot/src/host/browser/deepseek.js
deleted file mode 100644
index 97de85fe..00000000
--- a/ds-bot/src/host/browser/deepseek.js
+++ /dev/null
@@ -1,55 +0,0 @@
-// OpenDesign's OpenAI-compatible endpoint, model DeepSeek V4.1 Flash. The key
-// stays in the environment and is never written into a result, a log, or an error.
-const BASE = (process.env.OPEN_DESIGN_BASE_URL || 'https://amr-link.open-design.ai/v1').replace(/\/$/, '')
-const ENDPOINT = `${BASE}/chat/completions`
-export const MODEL = 'deepseek-v4.1-flash'
-
-export function deepseekKey() {
- return process.env.OPENDESIGN_API_KEY || process.env.DEEPSEEK_API_KEY || process.env.DEEPSEEK_KEY || ''
-}
-
-const stripImages = (messages) => messages.map(message => {
- if (!Array.isArray(message.content)) return message
- const text = message.content.filter(part => part.type === 'text').map(part => part.text).join('\n')
- return { ...message, content: text || '(screenshot omitted)' }
-})
-
-export async function completeDeepseek({ messages, tools, model = MODEL, timeoutMs = 90_000 }) {
- const key = deepseekKey()
- if (!key) throw new Error('OPENDESIGN_API_KEY is not set')
- const send = async (payload, signal) => {
- const response = await fetch(ENDPOINT, {
- method: 'POST',
- headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },
- body: JSON.stringify(payload),
- signal,
- })
- const body = await response.json().catch(() => ({}))
- if (!response.ok) {
- const detail = body?.error?.message || response.statusText
- throw new Error(`DeepSeek ${response.status}: ${detail}`)
- }
- return body
- }
- const payload = { model, messages, max_tokens: 4096 }
- if (tools?.length) payload.tools = tools
- const controller = new AbortController()
- const timer = setTimeout(() => controller.abort(), timeoutMs)
- try {
- const body = await send(payload, controller.signal)
- return { ...body, vision: 'accepted' }
- } catch (error) {
- const message = error instanceof Error ? error.message : String(error)
- if (!/image|multimodal|vision|unsupported/i.test(message)) throw error
- const retry = new AbortController()
- const retryTimer = setTimeout(() => retry.abort(), timeoutMs)
- try {
- const body = await send({ ...payload, messages: stripImages(messages) }, retry.signal)
- return { ...body, vision: 'rejected' }
- } finally {
- clearTimeout(retryTimer)
- }
- } finally {
- clearTimeout(timer)
- }
-}
diff --git a/ds-bot/src/host/browser/mount.js b/ds-bot/src/host/browser/mount.js
index fb3888dd..dbe5b4cd 100644
--- a/ds-bot/src/host/browser/mount.js
+++ b/ds-bot/src/host/browser/mount.js
@@ -18,9 +18,9 @@ export const browserOutput = {
/**
* @param {any} ctx
- * @param {{ toolDefs: any[], defineTool: Function, keyOf: (exec: any) => Promise<string|undefined>, profileRoot: string, downloadRoot?: string, uploadRoots?: (botId: string) => Promise<string[]>, headless: boolean, playwrightUrl?: string, executablePath?: string, actionTimeout?: number, authorize?: (exec: any, botId: string, action: any) => Promise<void>, complete?: Function }} options
+ * @param {{ toolDefs: any[], defineTool: Function, keyOf: (exec: any) => Promise<string|undefined>, profileRoot: string, downloadRoot?: string, uploadRoots?: (botId: string) => Promise<string[]>, headless: boolean, playwrightUrl?: string, executablePath?: string, actionTimeout?: number, authorize?: (exec: any, botId: string, action: any) => Promise<void>, completeFor?: (exec: any) => Function }} options
*/
-export function mountBrowserPlugin(ctx, { toolDefs, defineTool, keyOf, profileRoot, downloadRoot, uploadRoots, headless, playwrightUrl, executablePath, actionTimeout, authorize, complete }) {
+export function mountBrowserPlugin(ctx, { toolDefs, defineTool, keyOf, profileRoot, downloadRoot, uploadRoots, headless, playwrightUrl, executablePath, actionTimeout, authorize, completeFor }) {
const sessions = new Map()
ctx.effect(() => async () => {
const open = [...sessions.values()]
@@ -56,7 +56,7 @@ export function mountBrowserPlugin(ctx, { toolDefs, defineTool, keyOf, profileRo
botId,
profileDir: join(profileRoot, botId),
session,
- complete,
+ complete: completeFor?.(exec),
}))
},
}))
diff --git a/ds-bot/src/host/browser/session-model.js b/ds-bot/src/host/browser/session-model.js
new file mode 100644
index 00000000..7cadf7dc
--- /dev/null
+++ b/ds-bot/src/host/browser/session-model.js
@@ -0,0 +1,196 @@
+import {
+ BlockAssembler,
+ createAssistantMessage,
+ createSystemMessage,
+ createToolResultMessage,
+ createUserMessage,
+} from '@deepseek-ai/dsh-llm'
+
+const NO_SESSION_MODEL = 'browser_task needs a Bot session model'
+const OMITTED_IMAGE = '[Browser screenshot omitted.]'
+
+function routeOf(exec) {
+ const session = exec?.agent?.session
+ if (!session || typeof session.requestHeader !== 'function' || typeof session.id !== 'string') {
+ throw new Error(NO_SESSION_MODEL)
+ }
+ let header
+ try {
+ header = session.requestHeader()
+ } catch {
+ throw new Error(NO_SESSION_MODEL)
+ }
+ if (!header?.config?.provider || !header.config.model) throw new Error(NO_SESSION_MODEL)
+ return { session, header, sessionId: session.id }
+}
+
+function textParts(content) {
+ if (Array.isArray(content)) return content.filter(part => part?.type === 'text').map(part => part.text ?? '')
+ return [typeof content === 'string' ? content : String(content ?? '')]
+}
+
+function imageInput(part) {
+ const url = part?.image_url?.url
+ if (typeof url !== 'string') return undefined
+ const match = /^data:(image\/(?:png|jpeg|webp|gif));base64,([A-Za-z0-9+/]+={0,2})$/i.exec(url)
+ if (!match) return undefined
+ return { mediaType: match[1].toLowerCase(), data: Buffer.from(match[2], 'base64'), name: 'Browser screenshot' }
+}
+
+function attachmentsOf(ctx) {
+ try {
+ return ctx.get?.('attachments') ?? ctx.attachments
+ } catch {
+ return ctx.attachments
+ }
+}
+
+async function userContent(content, attachments) {
+ const parts = Array.isArray(content) ? content : [{ type: 'text', text: String(content ?? '') }]
+ const blocks = []
+ const pending = []
+ let sawImage = false
+ let rejectedImage = false
+ for (const part of parts) {
+ if (part?.type === 'text') {
+ blocks.push({ type: 'text', text: String(part.text ?? '') })
+ continue
+ }
+ if (part?.type === 'image_url') {
+ sawImage = true
+ const input = imageInput(part)
+ if (!input || typeof attachments?.saveImages !== 'function') {
+ blocks.push({ type: 'text', text: OMITTED_IMAGE })
+ rejectedImage = true
+ continue
+ }
+ pending.push({ index: blocks.length, input })
+ blocks.push(undefined)
+ continue
+ }
+ if (part?.type) blocks.push({ type: 'text', text: String(part.text ?? '') })
+ }
+ if (pending.length > 0) {
+ try {
+ const refs = await attachments.saveImages(pending.map(item => item.input))
+ pending.forEach((item, index) => {
+ blocks[item.index] = { type: 'image', attachment: refs[index] }
+ })
+ } catch {
+ pending.forEach(item => {
+ blocks[item.index] = { type: 'text', text: OMITTED_IMAGE }
+ })
+ rejectedImage = true
+ }
+ }
+ return { blocks: blocks.filter(Boolean), sawImage, rejectedImage }
+}
+
+function assistantContent(message, config) {
+ const blocks = textParts(message.content)
+ .filter(text => text !== '')
+ .map(text => ({ type: 'text', text }))
+ const reasoning = message.reasoning_content ?? message.reasoning
+ if (typeof reasoning === 'string' && reasoning !== '') blocks.push({ type: 'reasoning', text: reasoning })
+ for (const call of message.tool_calls ?? []) {
+ const fn = call.function ?? {}
+ const args = fn.arguments
+ blocks.push({
+ type: 'tool-call',
+ id: String(call.id ?? ''),
+ name: String(fn.name ?? ''),
+ arguments: typeof args === 'string' ? args : JSON.stringify(args ?? {}),
+ })
+ }
+ return createAssistantMessage({
+ content: blocks,
+ source: { provider: config.provider, model: config.model },
+ })
+}
+
+async function dshMessages(messages, attachments, config) {
+ const output = []
+ let sawImage = false
+ let rejectedImage = false
+ for (const message of messages ?? []) {
+ if (message.role === 'system') {
+ output.push(createSystemMessage(textParts(message.content).join('\n')))
+ } else if (message.role === 'user') {
+ const content = await userContent(message.content, attachments)
+ output.push(createUserMessage({ content: content.blocks, source: { kind: 'user' } }))
+ sawImage ||= content.sawImage
+ rejectedImage ||= content.rejectedImage
+ } else if (message.role === 'assistant') {
+ output.push(assistantContent(message, config))
+ } else if (message.role === 'tool') {
+ output.push(createToolResultMessage({
+ callId: String(message.tool_call_id ?? ''),
+ content: textParts(message.content).map(text => ({ type: 'text', text })),
+ isError: false,
+ }))
+ }
+ }
+ return { messages: output, sawImage, rejectedImage }
+}
+
+function driverTools(tools) {
+ return (tools ?? []).map(tool => {
+ const fn = tool.function ?? tool
+ return {
+ name: String(fn.name ?? ''),
+ description: String(fn.description ?? ''),
+ parameters: fn.parameters ?? {},
+ }
+ })
+}
+
+async function routeReadsImages(ctx, config, rt) {
+ if (rt?.config?.textOnlyModels?.includes?.(`${config.provider}/${config.model}`)) return false
+ if (typeof ctx.llm?.resolveModelInfo !== 'function') return true
+ try {
+ const info = await ctx.llm.resolveModelInfo(config.provider, config.model)
+ return info.inputModalities === undefined || info.inputModalities.includes('image')
+ } catch {
+ return true
+ }
+}
+
+export function sessionComplete(ctx, exec, rt) {
+ return async ({ messages, tools }) => {
+ const { header, sessionId } = routeOf(exec)
+ const converted = await dshMessages(messages, attachmentsOf(ctx), header.config)
+ const requestMessages = typeof rt?.tagUsage === 'function'
+ ? rt.tagUsage(converted.messages, { sessionId, kind: 'browser-task' })
+ : converted.messages
+ const assembler = new BlockAssembler()
+ for await (const chunk of ctx.llm.stream({
+ ...header.config,
+ messages: requestMessages,
+ tools: driverTools(tools),
+ sessionId,
+ signal: exec.signal,
+ })) assembler.push(chunk)
+ const finish = assembler.finish
+ if (finish.kind === 'error' || finish.kind === 'aborted') {
+ throw new Error(finish.failure?.message ?? finish.kind)
+ }
+ const blocks = assembler.blocks()
+ const text = blocks.filter(block => block.type === 'text').map(block => block.text).join('\n')
+ const reasoning = blocks.filter(block => block.type === 'reasoning').map(block => block.text).join('\n')
+ const toolCalls = blocks.filter(block => block.type === 'tool-call').map(block => ({
+ id: block.id,
+ type: 'function',
+ function: { name: block.name, arguments: block.arguments },
+ }))
+ const message = { content: text }
+ if (reasoning) message.reasoning_content = reasoning
+ if (toolCalls.length > 0) message.tool_calls = toolCalls
+ const vision = converted.sawImage
+ ? converted.rejectedImage || !await routeReadsImages(ctx, header.config, rt) ? 'rejected' : 'accepted'
+ : undefined
+ return {
+ choices: [{ message, finish_reason: toolCalls.length > 0 ? 'tool_calls' : finish.kind === 'max-tokens' ? 'length' : 'stop' }],
+ ...(vision ? { vision } : {}),
+ }
+ }
+}
diff --git a/ds-bot/src/host/browser/task.js b/ds-bot/src/host/browser/task.js
index 18040f4a..a6a555a8 100644
--- a/ds-bot/src/host/browser/task.js
+++ b/ds-bot/src/host/browser/task.js
@@ -1,6 +1,5 @@
// One goal in, one answer out. The plugin runs the ref tools itself and returns
// the step list, so a Bot can hand off a whole web task and still check what happened.
-import { completeDeepseek } from './deepseek.js'
import { runToolLoop } from './loop.js'
import { systemPrompt as refsPrompt, toolDefs as refTools } from './refs.js'
@@ -11,7 +10,8 @@ const driverTools = (session) => refTools.map(tool => ({
execute: (args) => tool.execute(args, { session, botId: session.botId, profileDir: session.userDataDir }),
}))
-export async function runTask({ session, goal, complete = completeDeepseek, maxSteps = 8 }) {
+export async function runTask({ session, goal, complete, maxSteps = 8 }) {
+ if (typeof complete !== 'function') throw new Error('browser_task needs a Bot session model')
const result = await runToolLoop({
tools: driverTools(session),
complete,
diff --git a/ds-bot/test/browser-plugins.test.js b/ds-bot/test/browser-plugins.test.js
index df421729..8e3042a0 100644
--- a/ds-bot/test/browser-plugins.test.js
+++ b/ds-bot/test/browser-plugins.test.js
@@ -4,6 +4,7 @@ import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { test } from 'node:test'
import { apply, createRuntime } from '../index.js'
+import { defaultBrowserHeadless } from '../src/host/browser-plugins.js'
import { gateScope } from '../src/host/browser/gate.js'
import { systemPrompt as refsPrompt, toolDefs as refsTools } from '../src/host/browser/refs.js'
import { fakeApproval, fakeContext } from './fake-ctx.js'
@@ -99,6 +100,62 @@ test('no browser plugin is mounted unless one is selected for the team', (t) =>
assert.equal(ctx.registeredTools.has('browser_task'), false)
})
+test('browser headless defaults only on Linux without a display', () => {
+ assert.equal(defaultBrowserHeadless('linux', { DISPLAY: ':0' }), false)
+ assert.equal(defaultBrowserHeadless('linux', {}), true)
+ assert.equal(defaultBrowserHeadless('darwin', {}), false)
+})
+
+test('browser_task uses the Bot session model route when no test completion is injected', async (t) => {
+ const { ctx, stop } = start({
+ browserPlugin: 'task',
+ browserPluginApi: {
+ id: 'task',
+ tools: [{
+ name: 'browser_task',
+ needsBrowser: false,
+ async execute(_args, toolCtx) {
+ const response = await toolCtx.complete({
+ messages: [{ role: 'user', content: 'Read the page.' }],
+ tools: [{
+ type: 'function',
+ function: { name: 'browser_snapshot', description: 'Read page', parameters: { type: 'object' } },
+ }],
+ })
+ return response.choices[0].message.content
+ },
+ }],
+ },
+ })
+ t.after(stop)
+ const requests = []
+ ctx.llm = {
+ async *stream(request) {
+ requests.push(request)
+ yield { type: 'text-delta', index: 0, text: 'Session-routed response.' }
+ yield { type: 'finish', reason: { kind: 'stop' } }
+ },
+ }
+ const session = {
+ id: 'session-b2',
+ header: { id: 'b2' },
+ requestHeader: () => ({
+ config: { provider: 'mock-provider', model: 'bot-session-model' },
+ tools: [{ name: 'session-private-tool', description: '', parameters: {} }],
+ }),
+ }
+
+ const value = await ctx.registeredTools.get('browser_task').execute({}, {
+ agent: { id: 'b2', session },
+ signal: new AbortController().signal,
+ })
+
+ assert.equal(value, 'Session-routed response.')
+ assert.equal(requests[0].model, 'bot-session-model')
+ assert.equal(requests[0].sessionId, 'session-b2')
+ assert.deepEqual(requests[0].tools.map(tool => tool.name), ['browser_snapshot'])
+})
+
test('the selected plugin is the same for every Bot, and profiles stay apart', async (t) => {
const seen = []
const { ctx, stop } = start({
diff --git a/ds-bot/test/session-model.test.js b/ds-bot/test/session-model.test.js
new file mode 100644
index 00000000..a40a2a82
--- /dev/null
+++ b/ds-bot/test/session-model.test.js
@@ -0,0 +1,168 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import { sessionComplete } from '../src/host/browser/session-model.js'
+
+const header = {
+ config: { provider: 'mock-provider', model: 'session-model', temperature: 0.25 },
+ tools: [{ name: 'session-private-tool', description: 'not for browser_task', parameters: {} }],
+}
+
+function fixture(chunks, { attachments, inputModalities } = {}) {
+ const requests = []
+ const tags = []
+ const ctx = {
+ get: name => name === 'attachments' ? attachments : undefined,
+ llm: {
+ async *stream(request) {
+ requests.push(request)
+ yield* chunks
+ },
+ async resolveModelInfo() {
+ return { inputModalities }
+ },
+ },
+ }
+ const session = {
+ id: 'session-browser-task',
+ requestHeader: () => header,
+ }
+ const rt = {
+ tagUsage(messages, options) {
+ tags.push(options)
+ return messages
+ },
+ }
+ return { ctx, exec: { agent: { session }, signal: new AbortController().signal }, rt, requests, tags }
+}
+
+const stop = { type: 'finish', reason: { kind: 'stop' } }
+
+test('session completion converts DSH text to an OpenAI chat response', async () => {
+ const setup = fixture([
+ { type: 'text-delta', index: 0, text: 'The page title is Example.' },
+ stop,
+ ])
+ const complete = sessionComplete(setup.ctx, setup.exec, setup.rt)
+ const result = await complete({
+ messages: [{ role: 'system', content: 'Use the browser.' }, { role: 'user', content: 'Read the title.' }],
+ tools: [],
+ })
+
+ assert.equal(result.choices[0].message.content, 'The page title is Example.')
+ assert.deepEqual(setup.tags, [{ sessionId: 'session-browser-task', kind: 'browser-task' }])
+})
+
+test('session completion preserves DSH tool-call identity, name, and JSON arguments', async () => {
+ const setup = fixture([
+ { type: 'tool-call-delta', index: 0, id: 'call-browser-1', name: 'browser_open', argumentsDelta: '{"url":"https://example.com"}' },
+ { type: 'finish', reason: { kind: 'stop' } },
+ ])
+ const result = await sessionComplete(setup.ctx, setup.exec, setup.rt)({
+ messages: [{ role: 'user', content: 'Open example.com.' }],
+ tools: [],
+ })
+
+ assert.deepEqual(result.choices[0].message.tool_calls, [{
+ id: 'call-browser-1',
+ type: 'function',
+ function: { name: 'browser_open', arguments: '{"url":"https://example.com"}' },
+ }])
+})
+
+test('follow-up assistant tool call and tool result are converted to DSH messages', async () => {
+ const setup = fixture([
+ { type: 'text-delta', index: 0, text: 'Done.' },
+ stop,
+ ])
+ const messages = [
+ { role: 'system', content: 'Use browser tools.' },
+ { role: 'user', content: 'Open the page.' },
+ {
+ role: 'assistant',
+ content: '',
+ tool_calls: [{
+ id: 'call-browser-2',
+ type: 'function',
+ function: { name: 'browser_open', arguments: '{"url":"https://example.com"}' },
+ }],
+ },
+ { role: 'tool', tool_call_id: 'call-browser-2', content: 'Opened https://example.com' },
+ ]
+ const result = await sessionComplete(setup.ctx, setup.exec, setup.rt)({
+ messages,
+ tools: [{
+ type: 'function',
+ function: { name: 'browser_snapshot', description: 'Read page', parameters: { type: 'object' } },
+ }],
+ })
+
+ assert.equal(result.choices[0].message.content, 'Done.')
+ const request = setup.requests[0]
+ assert.deepEqual(request.messages.map(message => message.role), ['system', 'user', 'assistant', 'tool'])
+ assert.deepEqual(request.messages[2].content, [{
+ type: 'tool-call',
+ id: 'call-browser-2',
+ name: 'browser_open',
+ arguments: '{"url":"https://example.com"}',
+ }])
+ assert.equal(request.messages[3].toolCallId, 'call-browser-2')
+ assert.deepEqual(request.messages[3].content, [{ type: 'text', text: 'Opened https://example.com' }])
+ assert.deepEqual(request.tools, [{
+ name: 'browser_snapshot',
+ description: 'Read page',
+ parameters: { type: 'object' },
+ }])
+ assert.equal(request.model, 'session-model')
+ assert.equal(request.provider, 'mock-provider')
+ assert.equal(request.sessionId, 'session-browser-task')
+ assert.deepEqual(setup.tags, [{ sessionId: 'session-browser-task', kind: 'browser-task' }])
+})
+
+test('browser screenshots become DSH image blocks when the attachment store is available', async () => {
+ const saved = []
+ const attachments = {
+ async saveImages(inputs) {
+ saved.push(...inputs)
+ return [{
+ attachmentId: 'sha256:image',
+ mediaType: 'image/jpeg',
+ bytes: inputs[0].data.length,
+ width: 1,
+ height: 1,
+ }]
+ },
+ }
+ const setup = fixture([stop], { attachments, inputModalities: ['text', 'image'] })
+ await sessionComplete(setup.ctx, setup.exec)({
+ messages: [{
+ role: 'user',
+ content: [
+ { type: 'text', text: 'Screenshot.' },
+ { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,/9j/4AA=' } },
+ ],
+ }],
+ tools: [],
+ })
+
+ assert.equal(saved.length, 1)
+ assert.equal(saved[0].mediaType, 'image/jpeg')
+ assert.deepEqual(setup.requests[0].messages[0].content[1], {
+ type: 'image',
+ attachment: {
+ attachmentId: 'sha256:image',
+ mediaType: 'image/jpeg',
+ bytes: saved[0].data.length,
+ width: 1,
+ height: 1,
+ },
+ })
+})
+
+test('missing Bot session model fails with a clear error', async () => {
+ const setup = fixture([stop])
+ const complete = sessionComplete(setup.ctx, { agent: { session: {} } })
+ await assert.rejects(
+ complete({ messages: [], tools: [] }),
+ { message: 'browser_task needs a Bot session model' },
+ )
+})