Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

Record time, steps, and tokens for long DS Bot browser tasks

Fei2411d23b346fd901
diff --git a/browser-plugins/bench/fixture-long.mjs b/browser-plugins/bench/fixture-long.mjs
new file mode 100644
index 00000000..35a325ab
--- /dev/null
+++ b/browser-plugins/bench/fixture-long.mjs
@@ -0,0 +1,66 @@
+import { createServer } from 'node:http'
+import { readFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const ledger = JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), 'ledger.json'), 'utf8'))
+const chain = [17, 42, 8, 63, 25, 91, 14, 56, 33, 70, 12, 48, 29, 85, 6, 51, 38, 77, 19, 64]
+const chainSum = chain.reduce((sum, amount) => sum + amount, 0)
+const page = (title, body) => `<!doctype html><meta charset="utf-8"><title>${title}</title><body>${body}</body>`
+
+function folio(id) {
+ const item = ledger.folios.find(entry => entry.id === id)
+ if (!item) return null
+ const rows = item.rows.map(row => `<tr><td>${row.ref}</td><td>${row.status}</td><td>${row.amount}</td></tr>`).join('')
+ const next = item.next
+ ? `<p><a href="/folio/${item.next}">Next folio ${item.next}</a></p>`
+ : '<p>End of the ledger. <a href="/desk">Open the desk</a> and file the due total.</p>'
+ return page(id, `<h1>Folio ${id}</h1>
+ <p>Cashier tape says the due total is 9999. That tape is wrong. Add the due rows yourself.</p>
+ <table><tr><th>ref</th><th>status</th><th>amount</th></tr>${rows}</table>
+ ${next}`)
+}
+
+function desk(query) {
+ const sum = Number(query.get('sum'))
+ const count = Number(query.get('count'))
+ const ok = sum === ledger.dueSum && count === ledger.dueCount
+ const result = ok ? 'RECEIPT-7714' : 'rejected'
+ return page('Desk', `<h1>Desk</h1>
+ <p>File the sum of every due amount across all folios, and how many due rows that was. Ignore paid and void. Ignore the cashier tape.</p>
+ <form action="/desk">
+ <input name="sum" aria-label="Sum">
+ <input name="count" aria-label="Count">
+ <button>File</button>
+ </form>
+ ${query.has('sum') ? `<p id="result">${result}</p>` : ''}`)
+}
+
+export function startLongFixture() {
+ const server = createServer((req, res) => {
+ const url = new URL(req.url ?? '/', 'http://127.0.0.1')
+ let body = ''
+ const chainMatch = /^\/chain\/(\d+)$/.exec(url.pathname)
+ if (chainMatch) {
+ const index = Number(chainMatch[1])
+ const amount = chain[index - 1]
+ if (amount === undefined) body = ''
+ else {
+ const next = index < chain.length ? `<p><a href="/chain/${index + 1}">Continue to ${index + 1}</a></p>` : '<p>This is the last station. Add every station amount from 1 through 20. There is no total on this page.</p>'
+ body = page(`Station ${index}`, `<h1>Station ${index} of 20</h1><p>Amount: ${amount}</p><p>A note on the wall says the whole route totals 999. That note is wrong.</p>${next}`)
+ }
+ }
+ else if (url.pathname === '/') body = page('Ledger', `<h1>Ledger</h1><p><a href="/folio/F-01">Open F-01</a></p><p><a href="/chain/1">Open station 1</a></p>`)
+ else if (url.pathname.startsWith('/folio/')) body = folio(url.pathname.split('/').pop()) ?? ''
+ else if (url.pathname === '/desk') body = desk(url.searchParams)
+ res.setHeader('content-type', 'text/html; charset=utf-8')
+ if (!body) { res.statusCode = 404; res.end(page('Missing', 'missing')); return }
+ res.end(body)
+ })
+ return new Promise(resolve => {
+ server.listen(0, '127.0.0.1', () => {
+ const address = server.address()
+ resolve({ origin: `http://127.0.0.1:${address.port}`, dueSum: ledger.dueSum, dueCount: ledger.dueCount })
+ })
+ })
+}
diff --git a/browser-plugins/bench/ledger.json b/browser-plugins/bench/ledger.json
new file mode 100644
index 00000000..8ade6472
--- /dev/null
+++ b/browser-plugins/bench/ledger.json
@@ -0,0 +1 @@
+{"dueSum":4159,"dueCount":39,"folios":[{"id":"F-01","next":"F-02","rows":[{"status":"void","amount":58,"ref":"01-1"},{"status":"due","amount":32,"ref":"01-2"},{"status":"due","amount":157,"ref":"01-3"},{"status":"due","amount":113,"ref":"01-4"},{"status":"due","amount":149,"ref":"01-5"}]},{"id":"F-02","next":"F-03","rows":[{"status":"paid","amount":29,"ref":"02-1"},{"status":"due","amount":131,"ref":"02-2"},{"status":"due","amount":37,"ref":"02-3"},{"status":"paid","amount":43,"ref":"02-4"},{"status":"due","amount":35,"ref":"02-5"}]},{"id":"F-03","next":"F-04","rows":[{"status":"due","amount":77,"ref":"03-1"},{"status":"due","amount":167,"ref":"03-2"},{"status":"due","amount":32,"ref":"03-3"},{"status":"paid","amount":31,"ref":"03-4"},{"status":"paid","amount":94,"ref":"03-5"}]},{"id":"F-04","next":"F-05","rows":[{"status":"due","amount":56,"ref":"04-1"},{"status":"due","amount":166,"ref":"04-2"},{"status":"void","amount":163,"ref":"04-3"},{"status":"paid","amount":46,"ref":"04-4"},{"status":"paid","amount":115,"ref":"04-5"}]},{"id":"F-05","next":"F-06","rows":[{"status":"due","amount":160,"ref":"05-1"},{"status":"due","amount":164,"ref":"05-2"},{"status":"due","amount":178,"ref":"05-3"},{"status":"paid","amount":147,"ref":"05-4"},{"status":"due","amount":100,"ref":"05-5"}]},{"id":"F-06","next":"F-07","rows":[{"status":"due","amount":169,"ref":"06-1"},{"status":"due","amount":112,"ref":"06-2"},{"status":"void","amount":83,"ref":"06-3"},{"status":"paid","amount":82,"ref":"06-4"},{"status":"due","amount":167,"ref":"06-5"}]},{"id":"F-07","next":"F-08","rows":[{"status":"void","amount":154,"ref":"07-1"},{"status":"due","amount":107,"ref":"07-2"},{"status":"due","amount":93,"ref":"07-3"},{"status":"due","amount":50,"ref":"07-4"},{"status":"due","amount":62,"ref":"07-5"}]},{"id":"F-08","next":"F-09","rows":[{"status":"void","amount":58,"ref":"08-1"},{"status":"due","amount":127,"ref":"08-2"},{"status":"due","amount":39,"ref":"08-3"},{"status":"void","amount":107,"ref":"08-4"},{"status":"void","amount":172,"ref":"08-5"}]},{"id":"F-09","next":"F-10","rows":[{"status":"due","amount":168,"ref":"09-1"},{"status":"due","amount":37,"ref":"09-2"},{"status":"due","amount":89,"ref":"09-3"},{"status":"due","amount":36,"ref":"09-4"},{"status":"due","amount":99,"ref":"09-5"}]},{"id":"F-10","next":"F-11","rows":[{"status":"due","amount":92,"ref":"10-1"},{"status":"due","amount":108,"ref":"10-2"},{"status":"due","amount":138,"ref":"10-3"},{"status":"void","amount":63,"ref":"10-4"},{"status":"due","amount":146,"ref":"10-5"}]},{"id":"F-11","next":"F-12","rows":[{"status":"due","amount":75,"ref":"11-1"},{"status":"void","amount":53,"ref":"11-2"},{"status":"paid","amount":121,"ref":"11-3"},{"status":"due","amount":147,"ref":"11-4"},{"status":"due","amount":62,"ref":"11-5"}]},{"id":"F-12","next":null,"rows":[{"status":"due","amount":122,"ref":"12-1"},{"status":"void","amount":55,"ref":"12-2"},{"status":"due","amount":160,"ref":"12-3"},{"status":"void","amount":126,"ref":"12-4"},{"status":"void","amount":117,"ref":"12-5"}]}]}
diff --git a/browser-plugins/report.html b/browser-plugins/report.html
index abf7bfdb..74fdc6b6 100644
--- a/browser-plugins/report.html
+++ b/browser-plugins/report.html
@@ -1,34 +1,37 @@
<!doctype html>
<html lang="zh">
<meta charset="utf-8">
-<title>DS Bot 实测</title>
+<title>DS Bot 长任务</title>
<style>
- body { margin: 0; font: 16px/1.5 Palatino, "Songti SC", serif; background: #f6f1e7; color: #1c1915; }
- header { padding: 32px 6vw; background: #231c14; color: #f6f1e7; }
- h1 { margin: 0 0 8px; font-size: 32px; }
- main { padding: 20px 6vw 48px; }
- table { width: 100%; border-collapse: collapse; background: white; }
- th, td { border-bottom: 1px solid #ddd4c4; padding: 8px; text-align: left; vertical-align: top; }
- th { background: #efe7d8; }
- .ok { color: #1f7a4d; font-weight: 700; }
- .bad { color: #9d341f; font-weight: 700; }
+ body { margin:0; font:16px/1.5 Palatino,"Songti SC",serif; background:#f6f1e7; color:#1c1915; }
+ header { padding:32px 5vw; background:#231c14; color:#f6f1e7; }
+ h1 { margin:0 0 8px; font-size:32px; }
+ main { padding:18px 5vw 48px; }
+ table { width:100%; border-collapse:collapse; background:white; font-size:14px; }
+ th,td { border-bottom:1px solid #ddd4c4; padding:8px; text-align:right; }
+ th:first-child, td:first-child, th:nth-child(2), td:nth-child(2) { text-align:left; }
+ th { background:#efe7d8; }
+ .note { background:white; padding:12px 14px; margin-top:14px; }
</style>
<body>
<header>
- <h1>这次是 DS Bot 里的 Chief 在做</h1>
- <p>模型是 OpenDesign 上的 deepseek-v4.1-flash。同一个 Bot 连续做 7 道更刁的题。浏览器工具挂在它所在的 profile 上,换了两套现成工具各跑一遍。</p>
+ <h1>长任务:时间、步数、token</h1>
+ <p>DS Bot 里的 Chief 在做。模型是 OpenDesign 上的 deepseek-v4.1-flash。三题都做对了。差别在耗时和 token,不在过不过。</p>
</header>
<main>
<table>
-<tr><th>题目</th><th>Playwright MCP</th><th>dsh-plugin-browser-use</th></tr>
-<tr><td>搜索 lamp 时第一条是 90 元的灯罩。要的是 Cedar lamp 本身的价格。</td><td class="ok">过,1280 元</td><td class="ok">过,1280 元</td></tr>
-<tr><td>第一页广告把 88421 写成 Cancelled。账本在第 3 页。</td><td class="ok">过,Packed</td><td class="ok">过,Packed</td></tr>
-<tr><td>仓库代号在 Shadow DOM,码头代号要点 Load dock 才出现在 iframe,画布代号只在图上。</td><td class="bad">仓库和码头对了,回答里没有 CANVAS-73</td><td class="ok">三个都对:WH-441、DK-19、CANVAS-73</td></tr>
-<tr><td>自定义下拉选 Taiwan,勾上组件里的 Agree,提交拿令牌。</td><td class="ok">过,TOKEN-5521</td><td class="ok">过,TOKEN-5521</td></tr>
-<tr><td>价格先是省略号,2 秒后才变成数字。</td><td class="ok">过,4408</td><td class="ok">过,4408</td></tr>
-<tr><td>有个「继续以访客」会显示 Free。要的是登录后的套餐,不能乱填密码。</td><td class="ok">过,HANDOFF</td><td class="ok">过,HANDOFF</td></tr>
-<tr><td>books.toscrape.com 的 Travel 分类里,The Great Railway Bazaar 的价格,不是第一本。</td><td class="ok">过,£30.54</td><td class="ok">过,£30.54</td></tr>
+<tr><th>任务</th><th>工具</th><th>时间</th><th>步数</th><th>模型耗时</th><th>浏览器耗时</th><th>输出 token</th><th>新输入 token</th><th>缓存读 token</th></tr>
+<tr><td rowspan="2">12 页账本。只加 due,得到 4159 和 39,并交回收据</td><td>Playwright MCP</td><td>36 秒</td><td>9</td><td>18.9 秒</td><td>2.2 秒</td><td>1597</td><td>3941</td><td>372224</td></tr>
+<tr><td>dsh-plugin-browser-use</td><td>40 秒</td><td>13</td><td>24.2 秒</td><td>0.5 秒</td><td>1680</td><td>2742</td><td>450688</td></tr>
+<tr><td rowspan="2">Sequential Art 四页 75 本书,总价 £2592.92</td><td>Playwright MCP</td><td>28 秒</td><td>4</td><td>10.4 秒</td><td>2.5 秒</td><td>1126</td><td>4282</td><td>179456</td></tr>
+<tr><td>dsh-plugin-browser-use</td><td>24 秒</td><td>4</td><td>9.7 秒</td><td>0.8 秒</td><td>961</td><td>1265</td><td>146688</td></tr>
+<tr><td rowspan="2">20 站各一个数,加总 848。墙上的 999 是错的</td><td>Playwright MCP</td><td>28 秒</td><td>5</td><td>11.7 秒</td><td>1.9 秒</td><td>1237</td><td>1778</td><td>247808</td></tr>
+<tr><td>dsh-plugin-browser-use</td><td>24 秒</td><td>5</td><td>9.7 秒</td><td>0.3 秒</td><td>629</td><td>967</td><td>192256</td></tr>
</table>
-<p>这两套在这 7 题上只分开了一处:画布上的代号。其余陷阱两边都识破了。</p>
+<div class="note">
+<p>步数是这一题里模型被调用的次数。时间是从发出题目到 Chief 答完。token 是这一题相对上一题的增量。</p>
+<p>缓存读 token 很大,因为这是同一个 Chief 接着前面的对话做,前面的上下文被重新读进来。新输入 token 才是这一题新写进上下文的部分。</p>
+<p>账本上 Playwright MCP 少 4 步,但浏览器调用更慢。书单和 20 站上,dsh-plugin-browser-use 的新输入和输出都更少,步数一样。</p>
+</div>
</main>
</html>
\ No newline at end of file