A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Record time, steps, and tokens for long DS Bot browser tasks
Fei2411d23b346fd901
diff --git a/browser-plugins/bench/fixture-long.mjs b/browser-plugins/bench/fixture-long.mjsnew file mode 100644index 00000000..35a325ab--- /dev/null+++ b/browser-plugins/bench/fixture-long.mjs@@ -0,0 +1,66 @@+import { createServer } from 'node:http'+import { readFileSync } from 'node:fs'+import { dirname, join } from 'node:path'+import { fileURLToPath } from 'node:url'++const ledger = JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), 'ledger.json'), 'utf8'))+const chain = [17, 42, 8, 63, 25, 91, 14, 56, 33, 70, 12, 48, 29, 85, 6, 51, 38, 77, 19, 64]+const chainSum = chain.reduce((sum, amount) => sum + amount, 0)+const page = (title, body) => `<!doctype html><meta charset="utf-8"><title>${title}</title><body>${body}</body>`++function folio(id) {+ const item = ledger.folios.find(entry => entry.id === id)+ if (!item) return null+ const rows = item.rows.map(row => `<tr><td>${row.ref}</td><td>${row.status}</td><td>${row.amount}</td></tr>`).join('')+ const next = item.next+ ? `<p><a href="/folio/${item.next}">Next folio ${item.next}</a></p>`+ : '<p>End of the ledger. <a href="/desk">Open the desk</a> and file the due total.</p>'+ return page(id, `<h1>Folio ${id}</h1>+ <p>Cashier tape says the due total is 9999. That tape is wrong. Add the due rows yourself.</p>+ <table><tr><th>ref</th><th>status</th><th>amount</th></tr>${rows}</table>+ ${next}`)+}++function desk(query) {+ const sum = Number(query.get('sum'))+ const count = Number(query.get('count'))+ const ok = sum === ledger.dueSum && count === ledger.dueCount+ const result = ok ? 'RECEIPT-7714' : 'rejected'+ return page('Desk', `<h1>Desk</h1>+ <p>File the sum of every due amount across all folios, and how many due rows that was. Ignore paid and void. Ignore the cashier tape.</p>+ <form action="/desk">+ <input name="sum" aria-label="Sum">+ <input name="count" aria-label="Count">+ <button>File</button>+ </form>+ ${query.has('sum') ? `<p id="result">${result}</p>` : ''}`)+}++export function startLongFixture() {+ const server = createServer((req, res) => {+ const url = new URL(req.url ?? '/', 'http://127.0.0.1')+ let body = ''+ const chainMatch = /^\/chain\/(\d+)$/.exec(url.pathname)+ if (chainMatch) {+ const index = Number(chainMatch[1])+ const amount = chain[index - 1]+ if (amount === undefined) body = ''+ else {+ const next = index < chain.length ? `<p><a href="/chain/${index + 1}">Continue to ${index + 1}</a></p>` : '<p>This is the last station. Add every station amount from 1 through 20. There is no total on this page.</p>'+ body = page(`Station ${index}`, `<h1>Station ${index} of 20</h1><p>Amount: ${amount}</p><p>A note on the wall says the whole route totals 999. That note is wrong.</p>${next}`)+ }+ }+ else if (url.pathname === '/') body = page('Ledger', `<h1>Ledger</h1><p><a href="/folio/F-01">Open F-01</a></p><p><a href="/chain/1">Open station 1</a></p>`)+ else if (url.pathname.startsWith('/folio/')) body = folio(url.pathname.split('/').pop()) ?? ''+ else if (url.pathname === '/desk') body = desk(url.searchParams)+ res.setHeader('content-type', 'text/html; charset=utf-8')+ if (!body) { res.statusCode = 404; res.end(page('Missing', 'missing')); return }+ res.end(body)+ })+ return new Promise(resolve => {+ server.listen(0, '127.0.0.1', () => {+ const address = server.address()+ resolve({ origin: `http://127.0.0.1:${address.port}`, dueSum: ledger.dueSum, dueCount: ledger.dueCount })+ })+ })+}diff --git a/browser-plugins/bench/ledger.json b/browser-plugins/bench/ledger.jsonnew file mode 100644index 00000000..8ade6472--- /dev/null+++ b/browser-plugins/bench/ledger.json@@ -0,0 +1 @@+{"dueSum":4159,"dueCount":39,"folios":[{"id":"F-01","next":"F-02","rows":[{"status":"void","amount":58,"ref":"01-1"},{"status":"due","amount":32,"ref":"01-2"},{"status":"due","amount":157,"ref":"01-3"},{"status":"due","amount":113,"ref":"01-4"},{"status":"due","amount":149,"ref":"01-5"}]},{"id":"F-02","next":"F-03","rows":[{"status":"paid","amount":29,"ref":"02-1"},{"status":"due","amount":131,"ref":"02-2"},{"status":"due","amount":37,"ref":"02-3"},{"status":"paid","amount":43,"ref":"02-4"},{"status":"due","amount":35,"ref":"02-5"}]},{"id":"F-03","next":"F-04","rows":[{"status":"due","amount":77,"ref":"03-1"},{"status":"due","amount":167,"ref":"03-2"},{"status":"due","amount":32,"ref":"03-3"},{"status":"paid","amount":31,"ref":"03-4"},{"status":"paid","amount":94,"ref":"03-5"}]},{"id":"F-04","next":"F-05","rows":[{"status":"due","amount":56,"ref":"04-1"},{"status":"due","amount":166,"ref":"04-2"},{"status":"void","amount":163,"ref":"04-3"},{"status":"paid","amount":46,"ref":"04-4"},{"status":"paid","amount":115,"ref":"04-5"}]},{"id":"F-05","next":"F-06","rows":[{"status":"due","amount":160,"ref":"05-1"},{"status":"due","amount":164,"ref":"05-2"},{"status":"due","amount":178,"ref":"05-3"},{"status":"paid","amount":147,"ref":"05-4"},{"status":"due","amount":100,"ref":"05-5"}]},{"id":"F-06","next":"F-07","rows":[{"status":"due","amount":169,"ref":"06-1"},{"status":"due","amount":112,"ref":"06-2"},{"status":"void","amount":83,"ref":"06-3"},{"status":"paid","amount":82,"ref":"06-4"},{"status":"due","amount":167,"ref":"06-5"}]},{"id":"F-07","next":"F-08","rows":[{"status":"void","amount":154,"ref":"07-1"},{"status":"due","amount":107,"ref":"07-2"},{"status":"due","amount":93,"ref":"07-3"},{"status":"due","amount":50,"ref":"07-4"},{"status":"due","amount":62,"ref":"07-5"}]},{"id":"F-08","next":"F-09","rows":[{"status":"void","amount":58,"ref":"08-1"},{"status":"due","amount":127,"ref":"08-2"},{"status":"due","amount":39,"ref":"08-3"},{"status":"void","amount":107,"ref":"08-4"},{"status":"void","amount":172,"ref":"08-5"}]},{"id":"F-09","next":"F-10","rows":[{"status":"due","amount":168,"ref":"09-1"},{"status":"due","amount":37,"ref":"09-2"},{"status":"due","amount":89,"ref":"09-3"},{"status":"due","amount":36,"ref":"09-4"},{"status":"due","amount":99,"ref":"09-5"}]},{"id":"F-10","next":"F-11","rows":[{"status":"due","amount":92,"ref":"10-1"},{"status":"due","amount":108,"ref":"10-2"},{"status":"due","amount":138,"ref":"10-3"},{"status":"void","amount":63,"ref":"10-4"},{"status":"due","amount":146,"ref":"10-5"}]},{"id":"F-11","next":"F-12","rows":[{"status":"due","amount":75,"ref":"11-1"},{"status":"void","amount":53,"ref":"11-2"},{"status":"paid","amount":121,"ref":"11-3"},{"status":"due","amount":147,"ref":"11-4"},{"status":"due","amount":62,"ref":"11-5"}]},{"id":"F-12","next":null,"rows":[{"status":"due","amount":122,"ref":"12-1"},{"status":"void","amount":55,"ref":"12-2"},{"status":"due","amount":160,"ref":"12-3"},{"status":"void","amount":126,"ref":"12-4"},{"status":"void","amount":117,"ref":"12-5"}]}]}diff --git a/browser-plugins/report.html b/browser-plugins/report.htmlindex abf7bfdb..74fdc6b6 100644--- a/browser-plugins/report.html+++ b/browser-plugins/report.html@@ -1,34 +1,37 @@<!doctype html><html lang="zh"><meta charset="utf-8">-<title>DS Bot 实测</title>+<title>DS Bot 长任务</title><style>- body { margin: 0; font: 16px/1.5 Palatino, "Songti SC", serif; background: #f6f1e7; color: #1c1915; }- header { padding: 32px 6vw; background: #231c14; color: #f6f1e7; }- h1 { margin: 0 0 8px; font-size: 32px; }- main { padding: 20px 6vw 48px; }- table { width: 100%; border-collapse: collapse; background: white; }- th, td { border-bottom: 1px solid #ddd4c4; padding: 8px; text-align: left; vertical-align: top; }- th { background: #efe7d8; }- .ok { color: #1f7a4d; font-weight: 700; }- .bad { color: #9d341f; font-weight: 700; }+ body { margin:0; font:16px/1.5 Palatino,"Songti SC",serif; background:#f6f1e7; color:#1c1915; }+ header { padding:32px 5vw; background:#231c14; color:#f6f1e7; }+ h1 { margin:0 0 8px; font-size:32px; }+ main { padding:18px 5vw 48px; }+ table { width:100%; border-collapse:collapse; background:white; font-size:14px; }+ th,td { border-bottom:1px solid #ddd4c4; padding:8px; text-align:right; }+ th:first-child, td:first-child, th:nth-child(2), td:nth-child(2) { text-align:left; }+ th { background:#efe7d8; }+ .note { background:white; padding:12px 14px; margin-top:14px; }</style><body><header>- <h1>这次是 DS Bot 里的 Chief 在做</h1>- <p>模型是 OpenDesign 上的 deepseek-v4.1-flash。同一个 Bot 连续做 7 道更刁的题。浏览器工具挂在它所在的 profile 上,换了两套现成工具各跑一遍。</p>+ <h1>长任务:时间、步数、token</h1>+ <p>DS Bot 里的 Chief 在做。模型是 OpenDesign 上的 deepseek-v4.1-flash。三题都做对了。差别在耗时和 token,不在过不过。</p></header><main><table>-<tr><th>题目</th><th>Playwright MCP</th><th>dsh-plugin-browser-use</th></tr>-<tr><td>搜索 lamp 时第一条是 90 元的灯罩。要的是 Cedar lamp 本身的价格。</td><td class="ok">过,1280 元</td><td class="ok">过,1280 元</td></tr>-<tr><td>第一页广告把 88421 写成 Cancelled。账本在第 3 页。</td><td class="ok">过,Packed</td><td class="ok">过,Packed</td></tr>-<tr><td>仓库代号在 Shadow DOM,码头代号要点 Load dock 才出现在 iframe,画布代号只在图上。</td><td class="bad">仓库和码头对了,回答里没有 CANVAS-73</td><td class="ok">三个都对:WH-441、DK-19、CANVAS-73</td></tr>-<tr><td>自定义下拉选 Taiwan,勾上组件里的 Agree,提交拿令牌。</td><td class="ok">过,TOKEN-5521</td><td class="ok">过,TOKEN-5521</td></tr>-<tr><td>价格先是省略号,2 秒后才变成数字。</td><td class="ok">过,4408</td><td class="ok">过,4408</td></tr>-<tr><td>有个「继续以访客」会显示 Free。要的是登录后的套餐,不能乱填密码。</td><td class="ok">过,HANDOFF</td><td class="ok">过,HANDOFF</td></tr>-<tr><td>books.toscrape.com 的 Travel 分类里,The Great Railway Bazaar 的价格,不是第一本。</td><td class="ok">过,£30.54</td><td class="ok">过,£30.54</td></tr>+<tr><th>任务</th><th>工具</th><th>时间</th><th>步数</th><th>模型耗时</th><th>浏览器耗时</th><th>输出 token</th><th>新输入 token</th><th>缓存读 token</th></tr>+<tr><td rowspan="2">12 页账本。只加 due,得到 4159 和 39,并交回收据</td><td>Playwright MCP</td><td>36 秒</td><td>9</td><td>18.9 秒</td><td>2.2 秒</td><td>1597</td><td>3941</td><td>372224</td></tr>+<tr><td>dsh-plugin-browser-use</td><td>40 秒</td><td>13</td><td>24.2 秒</td><td>0.5 秒</td><td>1680</td><td>2742</td><td>450688</td></tr>+<tr><td rowspan="2">Sequential Art 四页 75 本书,总价 £2592.92</td><td>Playwright MCP</td><td>28 秒</td><td>4</td><td>10.4 秒</td><td>2.5 秒</td><td>1126</td><td>4282</td><td>179456</td></tr>+<tr><td>dsh-plugin-browser-use</td><td>24 秒</td><td>4</td><td>9.7 秒</td><td>0.8 秒</td><td>961</td><td>1265</td><td>146688</td></tr>+<tr><td rowspan="2">20 站各一个数,加总 848。墙上的 999 是错的</td><td>Playwright MCP</td><td>28 秒</td><td>5</td><td>11.7 秒</td><td>1.9 秒</td><td>1237</td><td>1778</td><td>247808</td></tr>+<tr><td>dsh-plugin-browser-use</td><td>24 秒</td><td>5</td><td>9.7 秒</td><td>0.3 秒</td><td>629</td><td>967</td><td>192256</td></tr></table>-<p>这两套在这 7 题上只分开了一处:画布上的代号。其余陷阱两边都识破了。</p>+<div class="note">+<p>步数是这一题里模型被调用的次数。时间是从发出题目到 Chief 答完。token 是这一题相对上一题的增量。</p>+<p>缓存读 token 很大,因为这是同一个 Chief 接着前面的对话做,前面的上下文被重新读进来。新输入 token 才是这一题新写进上下文的部分。</p>+<p>账本上 Playwright MCP 少 4 步,但浏览器调用更慢。书单和 20 站上,dsh-plugin-browser-use 的新输入和输出都更少,步数一样。</p>+</div></main></html>\ No newline at end of file