A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Route browser_task through the Bot session model
Fei2411305ec58ff617
diff --git a/browser-plugins/README.md b/browser-plugins/README.mdindex 6a95873e..7f217b2f 100644--- a/browser-plugins/README.md+++ b/browser-plugins/README.md@@ -20,6 +20,10 @@ node bench/run.mjsnode bench/render-report.mjs```-模型走 OpenDesign `https://amr-link.open-design.ai/v1`,模型 id 是 `deepseek-v4.1-flash`。密钥用环境变量 `OPENDESIGN_API_KEY`。不要写进仓库。+`browser_task` 使用调用它的 Bot 在 DSH 中配置的模型。跑基准测试前设置 `BENCH_BASE_URL`、`BENCH_MODEL` 和 `BENCH_API_KEY`。++```sh+BENCH_BASE_URL=... BENCH_MODEL=... BENCH_API_KEY=... node bench/run.mjs+```设计说明在 `docs/plans/2026-10-21-browser-plugins.md`。diff --git a/browser-plugins/bench/deepseek.js b/browser-plugins/bench/deepseek.jsnew file mode 100644index 00000000..6b8f3e51--- /dev/null+++ b/browser-plugins/bench/deepseek.js@@ -0,0 +1,55 @@+const ENDPOINT_PATH = '/chat/completions'++export function benchConfig() {+ return {+ baseUrl: process.env.BENCH_BASE_URL?.replace(/\/$/, ''),+ model: process.env.BENCH_MODEL,+ apiKey: process.env.BENCH_API_KEY,+ }+}++export async function completeBench({ messages, tools, timeoutMs = 90_000 }) {+ const { baseUrl, model, apiKey } = benchConfig()+ if (!baseUrl || !model || !apiKey) {+ throw new Error('Set BENCH_BASE_URL, BENCH_MODEL, and BENCH_API_KEY to run the browser benchmark.')+ }+ const send = async (payload, signal) => {+ const response = await fetch(`${baseUrl}${ENDPOINT_PATH}`, {+ method: 'POST',+ headers: { 'content-type': 'application/json', authorization: `Bearer ${apiKey}` },+ body: JSON.stringify(payload),+ signal,+ })+ const body = await response.json().catch(() => ({}))+ if (!response.ok) {+ const detail = body?.error?.message || response.statusText+ throw new Error(`Benchmark model ${response.status}: ${detail}`)+ }+ return body+ }+ const payload = { model, messages, max_tokens: 4096 }+ if (tools?.length) payload.tools = tools+ const controller = new AbortController()+ const timer = setTimeout(() => controller.abort(), timeoutMs)+ try {+ const body = await send(payload, controller.signal)+ return { ...body, vision: 'accepted' }+ } catch (error) {+ const message = error instanceof Error ? error.message : String(error)+ if (!/image|multimodal|vision|unsupported/i.test(message)) throw error+ const retry = new AbortController()+ const retryTimer = setTimeout(() => retry.abort(), timeoutMs)+ try {+ const stripped = messages.map(message => {+ if (!Array.isArray(message.content)) return message+ const text = message.content.filter(part => part.type === 'text').map(part => part.text).join('\n')+ return { ...message, content: text || '(screenshot omitted)' }+ })+ return { ...(await send({ ...payload, messages: stripped }, retry.signal)), vision: 'rejected' }+ } finally {+ clearTimeout(retryTimer)+ }+ } finally {+ clearTimeout(timer)+ }+}diff --git a/browser-plugins/bench/render-report.mjs b/browser-plugins/bench/render-report.mjsindex b4e5b77b..94f935ad 100644--- a/browser-plugins/bench/render-report.mjs+++ b/browser-plugins/bench/render-report.mjs@@ -10,7 +10,7 @@ const esc = (value) => String(value ?? '').replace(/>/g, '>').replace(/"/g, '"')-let results = { key: 'missing', driver: [], isolation: {}, plugins: {}, model: 'deepseek-v4.1-flash' }+let results = { key: 'missing', driver: [], isolation: {}, plugins: {}, model: process.env.BENCH_MODEL ?? '' }try {results = JSON.parse(await readFile(new URL('./results.json', outDir), 'utf8'))} catch { /* rendered before the bench */ }@@ -161,7 +161,7 @@ const html = `<!doctype html><section id="runs"><h2>实测</h2><p>标准用了公开的 <a href="https://github.com/MinorJerry/WebVoyager">WebVoyager</a>(browser-use 也用这套报过成绩)。整套六百多条、很多站要登录或已经过期,不全跑。抽了两条 arXiv 原文任务:<code>ArXiv--11</code>、<code>ArXiv--6</code>。另外用自建页面补 WebVoyager 不覆盖的部分:检索后再点进去、翻页、Shadow DOM、同源 iframe、只存在于 canvas 的字、登录必须交给人。</p>- <p>模型 <code>${esc(results.model || 'deepseek-v4.1-flash')}</code>,经 OpenDesign <code>https://amr-link.open-design.ai/v1</code>。密钥:${results.key === 'present' ? '用的是环境里的 OPENDESIGN_API_KEY,没有写入报告。' : '这台环境里没有 OPENDESIGN_API_KEY,模型任务没有跑。'}</p>+ <p>模型 <code>${esc(results.model || 'BENCH_MODEL')}</code>,地址由 <code>BENCH_BASE_URL</code> 配置。${results.key === 'present' ? '使用了环境变量 BENCH_API_KEY,没有写入报告。' : '模型任务需要配置 BENCH_BASE_URL、BENCH_MODEL 和 BENCH_API_KEY。'}</p><h3>驱动,不经过模型</h3><ul>${driverRows || '<li>还没有驱动结果</li>'}</ul><h3>两个 Bot 的 cookie</h3>diff --git a/browser-plugins/bench/run.mjs b/browser-plugins/bench/run.mjsindex 296a11f1..1c873c74 100644--- a/browser-plugins/bench/run.mjs+++ b/browser-plugins/bench/run.mjs@@ -1,7 +1,7 @@import { mkdir, writeFile } from 'node:fs/promises'import { tmpdir } from 'node:os'import { join } from 'node:path'-import { completeDeepseek, deepseekKey, MODEL } from '../../ds-bot/src/host/browser/deepseek.js'+import { benchConfig, completeBench } from './deepseek.js'import { runToolLoop } from '../../ds-bot/src/host/browser/loop.js'import * as refs from '../../ds-bot/src/host/browser/refs.js'import * as script from '../../ds-bot/src/host/browser/script.js'@@ -89,14 +89,14 @@ function bind(plugin, session) {name: tool.name,description: tool.description,parameters: tool.parameters,- execute: (args) => tool.execute(args, { session, botId: session.botId, profileDir: session.userDataDir, complete: completeDeepseek }),+ execute: (args) => tool.execute(args, { session, botId: session.botId, profileDir: session.userDataDir, complete: completeBench }),}))}async function judge(task, answer, pageText) {if (task.check) return { ok: false, detail: 'local tasks are judged in code' }try {- const response = await completeDeepseek({+ const response = await completeBench({messages: [{ role: 'system', content: 'Judge a web agent. Reply with JSON only: {"success":true|false,"reason":"..."}.' },{ role: 'user', content: `Task: ${task.goal('')}\nAgent answer:\n${answer || '(empty)'}\nFinal page text:\n${pageText.slice(0, 4000)}` },@@ -115,10 +115,10 @@ async function runModelTask(plugin, task, session, fixture) {const started = Date.now()try {const result = plugin.id === 'task'- ? await runTask({ session, goal, complete: completeDeepseek, maxSteps: task.maxSteps })+ ? await runTask({ session, goal, complete: completeBench, maxSteps: task.maxSteps }): await runToolLoop({tools: bind(plugin, session),- complete: completeDeepseek,+ complete: completeBench,goal,system: plugin.systemPrompt,maxSteps: task.maxSteps,@@ -188,18 +188,18 @@ const driverSession = await openBotBrowser({ botId: 'driver', profileRoot, headlconst driver = await drive(driverSession, fixture)await driverSession.close()-const key = deepseekKey()+const { baseUrl, model, apiKey } = benchConfig()const results = {- model: MODEL,- endpoint: 'https://amr-link.open-design.ai/v1',- key: key ? 'present' : 'missing',+ model: model ?? '',+ endpoint: baseUrl ?? '',+ key: apiKey ? 'present' : 'missing',driver,isolation: await isolate(playwrightUrl),plugins: {},}-if (!key) {- for (const name of Object.keys(selectedPlugins)) results.plugins[name] = { skipped: 'OPENDESIGN_API_KEY is not set in this environment' }+if (!baseUrl || !model || !apiKey) {+ for (const name of Object.keys(selectedPlugins)) results.plugins[name] = { skipped: 'Set BENCH_BASE_URL, BENCH_MODEL, and BENCH_API_KEY to run model tasks.' }} else {for (const [name, plugin] of Object.entries(selectedPlugins)) {const rows = []diff --git a/browser-plugins/bench/upstream_browser_use.py b/browser-plugins/bench/upstream_browser_use.pyindex d7bf8335..74ef2b37 100644--- a/browser-plugins/bench/upstream_browser_use.py+++ b/browser-plugins/bench/upstream_browser_use.py@@ -70,9 +70,9 @@ async def one(llm, task_id, prompt, check):async def main():llm = ChatOpenAI(- model='deepseek-v4.1-flash',- base_url='https://amr-link.open-design.ai/v1',- api_key=os.environ['OPENDESIGN_API_KEY'],+ model=os.environ['BENCH_MODEL'],+ base_url=os.environ['BENCH_BASE_URL'],+ api_key=os.environ['BENCH_API_KEY'],dont_force_structured_output=True,timeout=90,)diff --git a/browser-plugins/bench/upstream_tools.mjs b/browser-plugins/bench/upstream_tools.mjsindex f7ee7a63..9f690ecf 100644--- a/browser-plugins/bench/upstream_tools.mjs+++ b/browser-plugins/bench/upstream_tools.mjs@@ -1,8 +1,8 @@-// Drive two real browser stacks with DeepSeek V4.1 Flash:+// Drive two real browser stacks with the configured benchmark model:// Microsoft @playwright/mcp, and the published npm package dsh-plugin-browser-use.import { spawn } from 'node:child_process'import { readFile } from 'node:fs/promises'-import { completeDeepseek } from '../../ds-bot/src/host/browser/deepseek.js'+import { completeBench, benchConfig } from './deepseek.js'const ORIGIN = process.env.FIXTURE_ORIGINconst CHROME = process.env.CHROME_PATH@@ -31,7 +31,7 @@ async function runLoop({ tools, call, goal, maxSteps = 8 }) {]const trace = []for (let step = 0; step < maxSteps; step += 1) {- const response = await completeDeepseek({ messages, tools })+ const response = await completeBench({ messages, tools })const message = response.choices?.[0]?.message ?? {}const calls = message.tool_calls ?? []messages.push({@@ -257,7 +257,8 @@ async function runDsh() {const { writeFile } = await import('node:fs/promises')const outPath = new URL('./out/upstream-tools.json', import.meta.url)-const out = { model: 'deepseek-v4.1-flash', endpoint: 'https://amr-link.open-design.ai/v1' }+const { model, baseUrl } = benchConfig()+const out = { model: model ?? '', endpoint: baseUrl ?? '' }const save = () => writeFile(outPath, JSON.stringify(out, null, 2))if (which.includes('mcp')) {try { out.playwrightMcp = await runMcp() } catch (error) { out.playwrightMcpError = String(error).slice(0, 500) }diff --git a/browser-plugins/refs/index.js b/browser-plugins/refs/index.jsindex 57f1fc14..19a358a9 100644--- a/browser-plugins/refs/index.js+++ b/browser-plugins/refs/index.js@@ -1,7 +1,6 @@import { join } from 'node:path'import { tmpdir } from 'node:os'import { defineTool } from '@deepseek-ai/dsh-tools'-import { completeDeepseek } from '../../ds-bot/src/host/browser/deepseek.js'import { mountBrowserPlugin } from '../../ds-bot/src/host/browser/mount.js'import { id, label, systemPrompt, toolDefs } from '../../ds-bot/src/host/browser/refs.js'@@ -17,7 +16,6 @@ export function apply(ctx, config = {}) {profileRoot: config.profileRoot ?? join(tmpdir(), 'ds-bot-browser-refs'),headless: config.headless !== false,playwrightUrl: config.playwrightUrl,- complete: completeDeepseek,async keyOf(exec) {return config.botId ?? exec.agent?.session?.header?.id ?? exec.agent?.id ?? 'session'},diff --git a/browser-plugins/script/index.js b/browser-plugins/script/index.jsindex 6652889d..994af35c 100644--- a/browser-plugins/script/index.js+++ b/browser-plugins/script/index.js@@ -1,7 +1,6 @@import { join } from 'node:path'import { tmpdir } from 'node:os'import { defineTool } from '@deepseek-ai/dsh-tools'-import { completeDeepseek } from '../../ds-bot/src/host/browser/deepseek.js'import { mountBrowserPlugin } from '../../ds-bot/src/host/browser/mount.js'import { id, label, systemPrompt, toolDefs } from '../../ds-bot/src/host/browser/script.js'@@ -15,7 +14,6 @@ export function apply(ctx, config = {}) {profileRoot: config.profileRoot ?? join(tmpdir(), 'ds-bot-browser-script'),headless: config.headless !== false,playwrightUrl: config.playwrightUrl,- complete: completeDeepseek,async keyOf(exec) {return config.botId ?? exec.agent?.session?.header?.id ?? exec.agent?.id ?? 'session'},diff --git a/browser-plugins/task/index.js b/browser-plugins/task/index.jsindex 72902e19..3b15ff8b 100644--- a/browser-plugins/task/index.js+++ b/browser-plugins/task/index.js@@ -1,8 +1,8 @@import { join } from 'node:path'import { tmpdir } from 'node:os'import { defineTool } from '@deepseek-ai/dsh-tools'-import { completeDeepseek } from '../../ds-bot/src/host/browser/deepseek.js'import { mountBrowserPlugin } from '../../ds-bot/src/host/browser/mount.js'+import { sessionComplete } from '../../ds-bot/src/host/browser/session-model.js'import { id, label, systemPrompt, toolDefs } from '../../ds-bot/src/host/browser/task.js'export { id, label, systemPrompt, toolDefs }@@ -15,7 +15,7 @@ export function apply(ctx, config = {}) {profileRoot: config.profileRoot ?? join(tmpdir(), 'ds-bot-browser-task'),headless: config.headless !== false,playwrightUrl: config.playwrightUrl,- complete: completeDeepseek,+ completeFor: exec => sessionComplete(ctx, exec),async keyOf(exec) {return config.botId ?? exec.agent?.session?.header?.id ?? exec.agent?.id ?? 'session'},diff --git a/browser-plugins/test/write-safety.test.js b/browser-plugins/test/write-safety.test.jsindex e4c63c43..b0828989 100644--- a/browser-plugins/test/write-safety.test.js+++ b/browser-plugins/test/write-safety.test.js@@ -387,7 +387,7 @@ test('browser_task driver tools inherit the outer authorization gate', async (t)headless: true,playwrightUrl,actionTimeout: 500,- complete,+ completeFor: () => complete,authorize: async (_exec, _botId, action) => { gateActions.push(action) },})const tool = registered.get('browser_task')diff --git a/docs/plans/2026-10-21-browser-plugins.md b/docs/plans/2026-10-21-browser-plugins.mdindex ba74a866..fc074459 100644--- a/docs/plans/2026-10-21-browser-plugins.md+++ b/docs/plans/2026-10-21-browser-plugins.md@@ -47,5 +47,5 @@ Bot 是长期身份。登录应该跟着 Bot 走,而不是跟着某一段对## 变更- 2026-10-21:维护者定下全团队一个后端、每 Bot 一份 profile、cookie 不同步。三个插件和实测在同一分支,默认关闭。-- 2026-10-22:实测模型是 OpenDesign 上的 `deepseek-v4.1-flash`(`https://amr-link.open-design.ai/v1`),密钥用 `OPENDESIGN_API_KEY`。+- 2026-10-22:`browser_task` 使用调用 Bot 在 DSH 中配置的模型;基准测试模型由 `BENCH_BASE_URL`、`BENCH_MODEL` 和 `BENCH_API_KEY` 配置。- 2026-10-23:维护者定下权限规则:默认放行;发帖、上传、下载、删除按预授权或审批;删除强制多核对一轮;付款一律交给用户;预授权可设时限,在设置的浏览器页管理。diff --git a/ds-bot/src/host/browser-plugins.js b/ds-bot/src/host/browser-plugins.jsindex a3b1de92..2927504d 100644--- a/ds-bot/src/host/browser-plugins.js+++ b/ds-bot/src/host/browser-plugins.js@@ -2,8 +2,8 @@// directory, including the group Sessions where that Bot speaks.import { join } from 'node:path'import { defineTool } from '@deepseek-ai/dsh-tools'-import { completeDeepseek } from './browser/deepseek.js'import { mountBrowserPlugin } from './browser/mount.js'+import { sessionComplete } from './browser/session-model.js'import * as refs from './browser/refs.js'import * as script from './browser/script.js'import * as task from './browser/task.js'@@ -13,6 +13,10 @@ export const browserPlugins = { refs, script, task }const NAMES = Object.keys(browserPlugins)const GRANT_KINDS = new Set(['post', 'upload', 'download', 'delete'])+export function defaultBrowserHeadless(platform, env = process.env) {+ return platform === 'linux' && !env.DISPLAY && !env.WAYLAND_DISPLAY+}+function optionalString(config, key) {const value = config[key]if (value !== undefined && typeof value !== 'string') throw new TypeError(`${key} must be a string`)@@ -88,7 +92,7 @@ export function install(rt) {const executablePath = optionalString(rt.config, 'browserExecutablePath')const headless = typeof rt.config.browserHeadless === 'boolean'? rt.config.browserHeadless- : !process.env.DISPLAY+ : defaultBrowserHeadless(process.platform)const downloadRoot = join(rt.home, 'browser-downloads')mountBrowserPlugin(rt.ctx, {toolDefs: plugin.tools ?? plugin.toolDefs,@@ -105,7 +109,9 @@ export function install(rt) {playwrightUrl,executablePath,authorize: (exec, botId, action) => authorizeBrowserAction(rt, exec, botId, action),- complete: rt.config.browserComplete ?? completeDeepseek,+ completeFor: rt.config.browserComplete+ ? () => rt.config.browserComplete+ : exec => sessionComplete(rt.ctx, exec, rt),async keyOf(exec) {await rt.load()const id = rt.selfOf(exec.agent?.session?.header?.id ?? exec.agent?.id)diff --git a/ds-bot/src/host/browser/deepseek.js b/ds-bot/src/host/browser/deepseek.jsdeleted file mode 100644index 97de85fe..00000000--- a/ds-bot/src/host/browser/deepseek.js+++ /dev/null@@ -1,55 +0,0 @@-// OpenDesign's OpenAI-compatible endpoint, model DeepSeek V4.1 Flash. The key-// stays in the environment and is never written into a result, a log, or an error.-const BASE = (process.env.OPEN_DESIGN_BASE_URL || 'https://amr-link.open-design.ai/v1').replace(/\/$/, '')-const ENDPOINT = `${BASE}/chat/completions`-export const MODEL = 'deepseek-v4.1-flash'--export function deepseekKey() {- return process.env.OPENDESIGN_API_KEY || process.env.DEEPSEEK_API_KEY || process.env.DEEPSEEK_KEY || ''-}--const stripImages = (messages) => messages.map(message => {- if (!Array.isArray(message.content)) return message- const text = message.content.filter(part => part.type === 'text').map(part => part.text).join('\n')- return { ...message, content: text || '(screenshot omitted)' }-})--export async function completeDeepseek({ messages, tools, model = MODEL, timeoutMs = 90_000 }) {- const key = deepseekKey()- if (!key) throw new Error('OPENDESIGN_API_KEY is not set')- const send = async (payload, signal) => {- const response = await fetch(ENDPOINT, {- method: 'POST',- headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },- body: JSON.stringify(payload),- signal,- })- const body = await response.json().catch(() => ({}))- if (!response.ok) {- const detail = body?.error?.message || response.statusText- throw new Error(`DeepSeek ${response.status}: ${detail}`)- }- return body- }- const payload = { model, messages, max_tokens: 4096 }- if (tools?.length) payload.tools = tools- const controller = new AbortController()- const timer = setTimeout(() => controller.abort(), timeoutMs)- try {- const body = await send(payload, controller.signal)- return { ...body, vision: 'accepted' }- } catch (error) {- const message = error instanceof Error ? error.message : String(error)- if (!/image|multimodal|vision|unsupported/i.test(message)) throw error- const retry = new AbortController()- const retryTimer = setTimeout(() => retry.abort(), timeoutMs)- try {- const body = await send({ ...payload, messages: stripImages(messages) }, retry.signal)- return { ...body, vision: 'rejected' }- } finally {- clearTimeout(retryTimer)- }- } finally {- clearTimeout(timer)- }-}diff --git a/ds-bot/src/host/browser/mount.js b/ds-bot/src/host/browser/mount.jsindex fb3888dd..dbe5b4cd 100644--- a/ds-bot/src/host/browser/mount.js+++ b/ds-bot/src/host/browser/mount.js@@ -18,9 +18,9 @@ export const browserOutput = {/*** @param {any} ctx- * @param {{ toolDefs: any[], defineTool: Function, keyOf: (exec: any) => Promise<string|undefined>, profileRoot: string, downloadRoot?: string, uploadRoots?: (botId: string) => Promise<string[]>, headless: boolean, playwrightUrl?: string, executablePath?: string, actionTimeout?: number, authorize?: (exec: any, botId: string, action: any) => Promise<void>, complete?: Function }} options+ * @param {{ toolDefs: any[], defineTool: Function, keyOf: (exec: any) => Promise<string|undefined>, profileRoot: string, downloadRoot?: string, uploadRoots?: (botId: string) => Promise<string[]>, headless: boolean, playwrightUrl?: string, executablePath?: string, actionTimeout?: number, authorize?: (exec: any, botId: string, action: any) => Promise<void>, completeFor?: (exec: any) => Function }} options*/-export function mountBrowserPlugin(ctx, { toolDefs, defineTool, keyOf, profileRoot, downloadRoot, uploadRoots, headless, playwrightUrl, executablePath, actionTimeout, authorize, complete }) {+export function mountBrowserPlugin(ctx, { toolDefs, defineTool, keyOf, profileRoot, downloadRoot, uploadRoots, headless, playwrightUrl, executablePath, actionTimeout, authorize, completeFor }) {const sessions = new Map()ctx.effect(() => async () => {const open = [...sessions.values()]@@ -56,7 +56,7 @@ export function mountBrowserPlugin(ctx, { toolDefs, defineTool, keyOf, profileRobotId,profileDir: join(profileRoot, botId),session,- complete,+ complete: completeFor?.(exec),}))},}))diff --git a/ds-bot/src/host/browser/session-model.js b/ds-bot/src/host/browser/session-model.jsnew file mode 100644index 00000000..7cadf7dc--- /dev/null+++ b/ds-bot/src/host/browser/session-model.js@@ -0,0 +1,196 @@+import {+ BlockAssembler,+ createAssistantMessage,+ createSystemMessage,+ createToolResultMessage,+ createUserMessage,+} from '@deepseek-ai/dsh-llm'++const NO_SESSION_MODEL = 'browser_task needs a Bot session model'+const OMITTED_IMAGE = '[Browser screenshot omitted.]'++function routeOf(exec) {+ const session = exec?.agent?.session+ if (!session || typeof session.requestHeader !== 'function' || typeof session.id !== 'string') {+ throw new Error(NO_SESSION_MODEL)+ }+ let header+ try {+ header = session.requestHeader()+ } catch {+ throw new Error(NO_SESSION_MODEL)+ }+ if (!header?.config?.provider || !header.config.model) throw new Error(NO_SESSION_MODEL)+ return { session, header, sessionId: session.id }+}++function textParts(content) {+ if (Array.isArray(content)) return content.filter(part => part?.type === 'text').map(part => part.text ?? '')+ return [typeof content === 'string' ? content : String(content ?? '')]+}++function imageInput(part) {+ const url = part?.image_url?.url+ if (typeof url !== 'string') return undefined+ const match = /^data:(image\/(?:png|jpeg|webp|gif));base64,([A-Za-z0-9+/]+={0,2})$/i.exec(url)+ if (!match) return undefined+ return { mediaType: match[1].toLowerCase(), data: Buffer.from(match[2], 'base64'), name: 'Browser screenshot' }+}++function attachmentsOf(ctx) {+ try {+ return ctx.get?.('attachments') ?? ctx.attachments+ } catch {+ return ctx.attachments+ }+}++async function userContent(content, attachments) {+ const parts = Array.isArray(content) ? content : [{ type: 'text', text: String(content ?? '') }]+ const blocks = []+ const pending = []+ let sawImage = false+ let rejectedImage = false+ for (const part of parts) {+ if (part?.type === 'text') {+ blocks.push({ type: 'text', text: String(part.text ?? '') })+ continue+ }+ if (part?.type === 'image_url') {+ sawImage = true+ const input = imageInput(part)+ if (!input || typeof attachments?.saveImages !== 'function') {+ blocks.push({ type: 'text', text: OMITTED_IMAGE })+ rejectedImage = true+ continue+ }+ pending.push({ index: blocks.length, input })+ blocks.push(undefined)+ continue+ }+ if (part?.type) blocks.push({ type: 'text', text: String(part.text ?? '') })+ }+ if (pending.length > 0) {+ try {+ const refs = await attachments.saveImages(pending.map(item => item.input))+ pending.forEach((item, index) => {+ blocks[item.index] = { type: 'image', attachment: refs[index] }+ })+ } catch {+ pending.forEach(item => {+ blocks[item.index] = { type: 'text', text: OMITTED_IMAGE }+ })+ rejectedImage = true+ }+ }+ return { blocks: blocks.filter(Boolean), sawImage, rejectedImage }+}++function assistantContent(message, config) {+ const blocks = textParts(message.content)+ .filter(text => text !== '')+ .map(text => ({ type: 'text', text }))+ const reasoning = message.reasoning_content ?? message.reasoning+ if (typeof reasoning === 'string' && reasoning !== '') blocks.push({ type: 'reasoning', text: reasoning })+ for (const call of message.tool_calls ?? []) {+ const fn = call.function ?? {}+ const args = fn.arguments+ blocks.push({+ type: 'tool-call',+ id: String(call.id ?? ''),+ name: String(fn.name ?? ''),+ arguments: typeof args === 'string' ? args : JSON.stringify(args ?? {}),+ })+ }+ return createAssistantMessage({+ content: blocks,+ source: { provider: config.provider, model: config.model },+ })+}++async function dshMessages(messages, attachments, config) {+ const output = []+ let sawImage = false+ let rejectedImage = false+ for (const message of messages ?? []) {+ if (message.role === 'system') {+ output.push(createSystemMessage(textParts(message.content).join('\n')))+ } else if (message.role === 'user') {+ const content = await userContent(message.content, attachments)+ output.push(createUserMessage({ content: content.blocks, source: { kind: 'user' } }))+ sawImage ||= content.sawImage+ rejectedImage ||= content.rejectedImage+ } else if (message.role === 'assistant') {+ output.push(assistantContent(message, config))+ } else if (message.role === 'tool') {+ output.push(createToolResultMessage({+ callId: String(message.tool_call_id ?? ''),+ content: textParts(message.content).map(text => ({ type: 'text', text })),+ isError: false,+ }))+ }+ }+ return { messages: output, sawImage, rejectedImage }+}++function driverTools(tools) {+ return (tools ?? []).map(tool => {+ const fn = tool.function ?? tool+ return {+ name: String(fn.name ?? ''),+ description: String(fn.description ?? ''),+ parameters: fn.parameters ?? {},+ }+ })+}++async function routeReadsImages(ctx, config, rt) {+ if (rt?.config?.textOnlyModels?.includes?.(`${config.provider}/${config.model}`)) return false+ if (typeof ctx.llm?.resolveModelInfo !== 'function') return true+ try {+ const info = await ctx.llm.resolveModelInfo(config.provider, config.model)+ return info.inputModalities === undefined || info.inputModalities.includes('image')+ } catch {+ return true+ }+}++export function sessionComplete(ctx, exec, rt) {+ return async ({ messages, tools }) => {+ const { header, sessionId } = routeOf(exec)+ const converted = await dshMessages(messages, attachmentsOf(ctx), header.config)+ const requestMessages = typeof rt?.tagUsage === 'function'+ ? rt.tagUsage(converted.messages, { sessionId, kind: 'browser-task' })+ : converted.messages+ const assembler = new BlockAssembler()+ for await (const chunk of ctx.llm.stream({+ ...header.config,+ messages: requestMessages,+ tools: driverTools(tools),+ sessionId,+ signal: exec.signal,+ })) assembler.push(chunk)+ const finish = assembler.finish+ if (finish.kind === 'error' || finish.kind === 'aborted') {+ throw new Error(finish.failure?.message ?? finish.kind)+ }+ const blocks = assembler.blocks()+ const text = blocks.filter(block => block.type === 'text').map(block => block.text).join('\n')+ const reasoning = blocks.filter(block => block.type === 'reasoning').map(block => block.text).join('\n')+ const toolCalls = blocks.filter(block => block.type === 'tool-call').map(block => ({+ id: block.id,+ type: 'function',+ function: { name: block.name, arguments: block.arguments },+ }))+ const message = { content: text }+ if (reasoning) message.reasoning_content = reasoning+ if (toolCalls.length > 0) message.tool_calls = toolCalls+ const vision = converted.sawImage+ ? converted.rejectedImage || !await routeReadsImages(ctx, header.config, rt) ? 'rejected' : 'accepted'+ : undefined+ return {+ choices: [{ message, finish_reason: toolCalls.length > 0 ? 'tool_calls' : finish.kind === 'max-tokens' ? 'length' : 'stop' }],+ ...(vision ? { vision } : {}),+ }+ }+}diff --git a/ds-bot/src/host/browser/task.js b/ds-bot/src/host/browser/task.jsindex 18040f4a..a6a555a8 100644--- a/ds-bot/src/host/browser/task.js+++ b/ds-bot/src/host/browser/task.js@@ -1,6 +1,5 @@// One goal in, one answer out. The plugin runs the ref tools itself and returns// the step list, so a Bot can hand off a whole web task and still check what happened.-import { completeDeepseek } from './deepseek.js'import { runToolLoop } from './loop.js'import { systemPrompt as refsPrompt, toolDefs as refTools } from './refs.js'@@ -11,7 +10,8 @@ const driverTools = (session) => refTools.map(tool => ({execute: (args) => tool.execute(args, { session, botId: session.botId, profileDir: session.userDataDir }),}))-export async function runTask({ session, goal, complete = completeDeepseek, maxSteps = 8 }) {+export async function runTask({ session, goal, complete, maxSteps = 8 }) {+ if (typeof complete !== 'function') throw new Error('browser_task needs a Bot session model')const result = await runToolLoop({tools: driverTools(session),complete,diff --git a/ds-bot/test/browser-plugins.test.js b/ds-bot/test/browser-plugins.test.jsindex df421729..8e3042a0 100644--- a/ds-bot/test/browser-plugins.test.js+++ b/ds-bot/test/browser-plugins.test.js@@ -4,6 +4,7 @@ import { tmpdir } from 'node:os'import { join } from 'node:path'import { test } from 'node:test'import { apply, createRuntime } from '../index.js'+import { defaultBrowserHeadless } from '../src/host/browser-plugins.js'import { gateScope } from '../src/host/browser/gate.js'import { systemPrompt as refsPrompt, toolDefs as refsTools } from '../src/host/browser/refs.js'import { fakeApproval, fakeContext } from './fake-ctx.js'@@ -99,6 +100,62 @@ test('no browser plugin is mounted unless one is selected for the team', (t) =>assert.equal(ctx.registeredTools.has('browser_task'), false)})+test('browser headless defaults only on Linux without a display', () => {+ assert.equal(defaultBrowserHeadless('linux', { DISPLAY: ':0' }), false)+ assert.equal(defaultBrowserHeadless('linux', {}), true)+ assert.equal(defaultBrowserHeadless('darwin', {}), false)+})++test('browser_task uses the Bot session model route when no test completion is injected', async (t) => {+ const { ctx, stop } = start({+ browserPlugin: 'task',+ browserPluginApi: {+ id: 'task',+ tools: [{+ name: 'browser_task',+ needsBrowser: false,+ async execute(_args, toolCtx) {+ const response = await toolCtx.complete({+ messages: [{ role: 'user', content: 'Read the page.' }],+ tools: [{+ type: 'function',+ function: { name: 'browser_snapshot', description: 'Read page', parameters: { type: 'object' } },+ }],+ })+ return response.choices[0].message.content+ },+ }],+ },+ })+ t.after(stop)+ const requests = []+ ctx.llm = {+ async *stream(request) {+ requests.push(request)+ yield { type: 'text-delta', index: 0, text: 'Session-routed response.' }+ yield { type: 'finish', reason: { kind: 'stop' } }+ },+ }+ const session = {+ id: 'session-b2',+ header: { id: 'b2' },+ requestHeader: () => ({+ config: { provider: 'mock-provider', model: 'bot-session-model' },+ tools: [{ name: 'session-private-tool', description: '', parameters: {} }],+ }),+ }++ const value = await ctx.registeredTools.get('browser_task').execute({}, {+ agent: { id: 'b2', session },+ signal: new AbortController().signal,+ })++ assert.equal(value, 'Session-routed response.')+ assert.equal(requests[0].model, 'bot-session-model')+ assert.equal(requests[0].sessionId, 'session-b2')+ assert.deepEqual(requests[0].tools.map(tool => tool.name), ['browser_snapshot'])+})+test('the selected plugin is the same for every Bot, and profiles stay apart', async (t) => {const seen = []const { ctx, stop } = start({diff --git a/ds-bot/test/session-model.test.js b/ds-bot/test/session-model.test.jsnew file mode 100644index 00000000..a40a2a82--- /dev/null+++ b/ds-bot/test/session-model.test.js@@ -0,0 +1,168 @@+import assert from 'node:assert/strict'+import { test } from 'node:test'+import { sessionComplete } from '../src/host/browser/session-model.js'++const header = {+ config: { provider: 'mock-provider', model: 'session-model', temperature: 0.25 },+ tools: [{ name: 'session-private-tool', description: 'not for browser_task', parameters: {} }],+}++function fixture(chunks, { attachments, inputModalities } = {}) {+ const requests = []+ const tags = []+ const ctx = {+ get: name => name === 'attachments' ? attachments : undefined,+ llm: {+ async *stream(request) {+ requests.push(request)+ yield* chunks+ },+ async resolveModelInfo() {+ return { inputModalities }+ },+ },+ }+ const session = {+ id: 'session-browser-task',+ requestHeader: () => header,+ }+ const rt = {+ tagUsage(messages, options) {+ tags.push(options)+ return messages+ },+ }+ return { ctx, exec: { agent: { session }, signal: new AbortController().signal }, rt, requests, tags }+}++const stop = { type: 'finish', reason: { kind: 'stop' } }++test('session completion converts DSH text to an OpenAI chat response', async () => {+ const setup = fixture([+ { type: 'text-delta', index: 0, text: 'The page title is Example.' },+ stop,+ ])+ const complete = sessionComplete(setup.ctx, setup.exec, setup.rt)+ const result = await complete({+ messages: [{ role: 'system', content: 'Use the browser.' }, { role: 'user', content: 'Read the title.' }],+ tools: [],+ })++ assert.equal(result.choices[0].message.content, 'The page title is Example.')+ assert.deepEqual(setup.tags, [{ sessionId: 'session-browser-task', kind: 'browser-task' }])+})++test('session completion preserves DSH tool-call identity, name, and JSON arguments', async () => {+ const setup = fixture([+ { type: 'tool-call-delta', index: 0, id: 'call-browser-1', name: 'browser_open', argumentsDelta: '{"url":"https://example.com"}' },+ { type: 'finish', reason: { kind: 'stop' } },+ ])+ const result = await sessionComplete(setup.ctx, setup.exec, setup.rt)({+ messages: [{ role: 'user', content: 'Open example.com.' }],+ tools: [],+ })++ assert.deepEqual(result.choices[0].message.tool_calls, [{+ id: 'call-browser-1',+ type: 'function',+ function: { name: 'browser_open', arguments: '{"url":"https://example.com"}' },+ }])+})++test('follow-up assistant tool call and tool result are converted to DSH messages', async () => {+ const setup = fixture([+ { type: 'text-delta', index: 0, text: 'Done.' },+ stop,+ ])+ const messages = [+ { role: 'system', content: 'Use browser tools.' },+ { role: 'user', content: 'Open the page.' },+ {+ role: 'assistant',+ content: '',+ tool_calls: [{+ id: 'call-browser-2',+ type: 'function',+ function: { name: 'browser_open', arguments: '{"url":"https://example.com"}' },+ }],+ },+ { role: 'tool', tool_call_id: 'call-browser-2', content: 'Opened https://example.com' },+ ]+ const result = await sessionComplete(setup.ctx, setup.exec, setup.rt)({+ messages,+ tools: [{+ type: 'function',+ function: { name: 'browser_snapshot', description: 'Read page', parameters: { type: 'object' } },+ }],+ })++ assert.equal(result.choices[0].message.content, 'Done.')+ const request = setup.requests[0]+ assert.deepEqual(request.messages.map(message => message.role), ['system', 'user', 'assistant', 'tool'])+ assert.deepEqual(request.messages[2].content, [{+ type: 'tool-call',+ id: 'call-browser-2',+ name: 'browser_open',+ arguments: '{"url":"https://example.com"}',+ }])+ assert.equal(request.messages[3].toolCallId, 'call-browser-2')+ assert.deepEqual(request.messages[3].content, [{ type: 'text', text: 'Opened https://example.com' }])+ assert.deepEqual(request.tools, [{+ name: 'browser_snapshot',+ description: 'Read page',+ parameters: { type: 'object' },+ }])+ assert.equal(request.model, 'session-model')+ assert.equal(request.provider, 'mock-provider')+ assert.equal(request.sessionId, 'session-browser-task')+ assert.deepEqual(setup.tags, [{ sessionId: 'session-browser-task', kind: 'browser-task' }])+})++test('browser screenshots become DSH image blocks when the attachment store is available', async () => {+ const saved = []+ const attachments = {+ async saveImages(inputs) {+ saved.push(...inputs)+ return [{+ attachmentId: 'sha256:image',+ mediaType: 'image/jpeg',+ bytes: inputs[0].data.length,+ width: 1,+ height: 1,+ }]+ },+ }+ const setup = fixture([stop], { attachments, inputModalities: ['text', 'image'] })+ await sessionComplete(setup.ctx, setup.exec)({+ messages: [{+ role: 'user',+ content: [+ { type: 'text', text: 'Screenshot.' },+ { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,/9j/4AA=' } },+ ],+ }],+ tools: [],+ })++ assert.equal(saved.length, 1)+ assert.equal(saved[0].mediaType, 'image/jpeg')+ assert.deepEqual(setup.requests[0].messages[0].content[1], {+ type: 'image',+ attachment: {+ attachmentId: 'sha256:image',+ mediaType: 'image/jpeg',+ bytes: saved[0].data.length,+ width: 1,+ height: 1,+ },+ })+})++test('missing Bot session model fails with a clear error', async () => {+ const setup = fixture([stop])+ const complete = sessionComplete(setup.ctx, { agent: { session: {} } })+ await assert.rejects(+ complete({ messages: [], tools: [] }),+ { message: 'browser_task needs a Bot session model' },+ )+})