Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

9.5 KB
Raw Blame
import { mkdir, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { benchConfig, completeBench } from './deepseek.js'
import { runToolLoop } from '../../ds-bot/src/host/browser/loop.js'
import * as refs from '../../ds-bot/src/host/browser/refs.js'
import * as script from '../../ds-bot/src/host/browser/script.js'
import { openBotBrowser } from '../../ds-bot/src/host/browser/session.js'
import { runTask } from '../../ds-bot/src/host/browser/task.js'
import { startFixture } from './fixture.mjs'
import { tasks } from './tasks.mjs'

const outDir = new URL('./out/', import.meta.url)
const playwrightUrl = new URL('../node_modules/playwright/index.js', import.meta.url).href
const plugins = { refs, script, task: { id: 'task', label: 'Whole task', systemPrompt: '', toolDefs: [] } }

const refOf = (data, pattern) => data.items.find(item => pattern.test(`${item.name} ${item.role} ${item.type}`))?.ref

async function drive(session, fixture) {
  const checks = []
  const record = (id, ok, detail) => checks.push({ id, ok, detail })

  await session.open(`${fixture.origin}/`)
  let data = await session.snapshotData()
  const field = data.items.find(item => item.role === 'input' && /search/i.test(item.name))
  const button = data.items.find(item => item.role === 'button' && /search/i.test(item.name))
  await session.fill(field.ref, 'cedar lamp')
  await session.click(button.ref)
  data = await session.snapshotData()
  await session.click(refOf(data, /cedar lamp/i))
  data = await session.snapshotData()
  record('shop', data.text.includes('1280'), data.text.includes('1280') ? 'item page shows 1280' : data.text.slice(0, 180))

  await session.open(`${fixture.origin}/orders`)
  data = await session.snapshotData()
  await session.click(refOf(data, /next/i))
  data = await session.snapshotData()
  record('orders', /88421/.test(data.text) && /packed/i.test(data.text), 'page 2 text checked')

  await session.open(`${fixture.origin}/widget`)
  let text = ''
  for (let attempt = 0; attempt < 10; attempt += 1) {
    text = await session.snapshotText()
    if (text.includes('WH-441') && text.includes('DK-19')) break
    await new Promise(resolve => setTimeout(resolve, 200))
  }
  record('shadow', text.includes('WH-441') && text.includes('DK-19'), 'snapshot includes shadow and frame text')
  record('canvas-hidden', !text.includes('CANVAS-73'), text.includes('CANVAS-73') ? 'canvas text leaked into the snapshot' : 'canvas code is not in the text')
  const picture = await session.shot()
  if (picture.image) {
    await mkdir(outDir, { recursive: true })
    await writeFile(new URL('./canvas.jpg', outDir), Buffer.from(picture.image, 'base64'))
  }
  record('canvas-shot', Boolean(picture.image), picture.image ? 'jpeg saved' : picture.text)

  await session.open(`${fixture.origin}/account`)
  session.handoff('login', 'Sign in, then tell me to resume.')
  let blocked = false
  try {
    data = await session.snapshotData()
    await session.fill(refOf(data, /password/i), 'guessed-secret')
  } catch (error) {
    blocked = /waiting for the user/i.test(error.message)
  }
  record('login-handoff', blocked && fixture.posted.length === 0 && session.hold?.reason === 'login', blocked ? 'actions blocked, no password posted' : 'handoff did not block')
  session.resume()
  const scripted = await session.runScript(`
    await goto(${JSON.stringify(`${fixture.origin}/`)})
    let page = await snapshot()
    const read = (text) => [...text.matchAll(/\\[#(\\d+)\\] (\\S+) "([^"]*)"/g)]
    let refs = read(page)
    const field = refs.find(([, , role, name]) => role === 'input' && /search/i.test(name))
    const button = refs.find(([, , role, name]) => role.startsWith('button') && /search/i.test(name))
    await fill(Number(field[1]), 'cedar lamp')
    await click(Number(button[1]))
    page = await snapshot()
    const lamp = read(page).find(([, , , name]) => /cedar lamp/i.test(name))
    await click(Number(lamp[1]))
    page = await snapshot()
    if (!page.includes('1280')) throw new Error('price missing')
    await done('1280 yuan')
  `)
  record('script-shop', scripted.includes('1280'), scripted.slice(0, 180))
  return checks
}

function bind(plugin, session) {
  return plugin.toolDefs.map(tool => ({
    name: tool.name,
    description: tool.description,
    parameters: tool.parameters,
    execute: (args) => tool.execute(args, { session, botId: session.botId, profileDir: session.userDataDir, complete: completeBench }),
  }))
}

async function judge(task, answer, pageText) {
  if (task.check) return { ok: false, detail: 'local tasks are judged in code' }
  try {
    const response = await completeBench({
      messages: [
        { role: 'system', content: 'Judge a web agent. Reply with JSON only: {"success":true|false,"reason":"..."}.' },
        { role: 'user', content: `Task: ${task.goal('')}\nAgent answer:\n${answer || '(empty)'}\nFinal page text:\n${pageText.slice(0, 4000)}` },
      ],
    })
    const text = response.choices?.[0]?.message?.content ?? ''
    const parsed = JSON.parse(text.slice(text.indexOf('{'), text.lastIndexOf('}') + 1))
    return { ok: parsed.success === true, detail: String(parsed.reason ?? '').slice(0, 300) }
  } catch (error) {
    return { ok: false, detail: error instanceof Error ? error.message : String(error) }
  }
}

async function runModelTask(plugin, task, session, fixture) {
  const goal = task.goal(fixture.origin)
  const started = Date.now()
  try {
    const result = plugin.id === 'task'
      ? await runTask({ session, goal, complete: completeBench, maxSteps: task.maxSteps })
      : await runToolLoop({
        tools: bind(plugin, session),
        complete: completeBench,
        goal,
        system: plugin.systemPrompt,
        maxSteps: task.maxSteps,
      })
    const pageText = await session.snapshotText().catch(() => '')
    const localOk = task.check ? task.check(result.answer, { handoff: session.hold?.reason, passwords: fixture.posted }) : null
    const judged = task.check ? null : await judge(task, result.answer, pageText)
    return {
      id: task.id,
      ok: task.check ? localOk === true : judged?.ok === true,
      steps: result.steps,
      stopped: result.stopped,
      vision: result.vision,
      ms: Date.now() - started,
      answer: (result.answer || '').slice(0, 800),
      judge: judged?.detail ?? '',
      trace: result.trace.map(step => step.name),
    }
  } catch (error) {
    return {
      id: task.id,
      ok: false,
      steps: 0,
      ms: Date.now() - started,
      answer: '',
      error: error instanceof Error ? error.message : String(error),
      trace: [],
    }
  }
}

async function isolate(playwrightUrl) {
  const fixture = await startFixture()
  const profileRoot = join(tmpdir(), `bot-browser-${Date.now()}`)
  const open = (botId) => openBotBrowser({ botId, profileRoot, headless: true, playwrightUrl })
  const alpha = await open('alpha')
  await alpha.open(`${fixture.origin}/set-cookie`)
  await alpha.open(`${fixture.origin}/who`)
  const alphaText = await alpha.snapshotText()
  await alpha.close()
  const beta = await open('beta')
  await beta.open(`${fixture.origin}/who`)
  const betaText = await beta.snapshotText()
  await beta.close()
  const again = await open('alpha')
  await again.open(`${fixture.origin}/who`)
  const againText = await again.snapshotText()
  await again.close()
  await fixture.close()
  return {
    alphaSawCookie: alphaText.includes('bot=alpha'),
    betaSawCookie: betaText.includes('bot=alpha'),
    alphaKeptCookie: againText.includes('bot=alpha'),
  }
}

const onlyTasks = (process.env.BENCH_TASKS ?? '').split(',').map(item => item.trim()).filter(Boolean)
const onlyPlugins = (process.env.BENCH_PLUGINS ?? '').split(',').map(item => item.trim()).filter(Boolean)
const selectedTasks = onlyTasks.length ? tasks.filter(task => onlyTasks.includes(task.id)) : tasks
const selectedPlugins = onlyPlugins.length
  ? Object.fromEntries(Object.entries(plugins).filter(([name]) => onlyPlugins.includes(name)))
  : plugins

const fixture = await startFixture()
const profileRoot = join(tmpdir(), `bot-browser-bench-${Date.now()}`)
const driverSession = await openBotBrowser({ botId: 'driver', profileRoot, headless: true, playwrightUrl })
const driver = await drive(driverSession, fixture)
await driverSession.close()

const { baseUrl, model, apiKey } = benchConfig()
const results = {
  model: model ?? '',
  endpoint: baseUrl ?? '',
  key: apiKey ? 'present' : 'missing',
  driver,
  isolation: await isolate(playwrightUrl),
  plugins: {},
}

if (!baseUrl || !model || !apiKey) {
  for (const name of Object.keys(selectedPlugins)) results.plugins[name] = { skipped: 'Set BENCH_BASE_URL, BENCH_MODEL, and BENCH_API_KEY to run model tasks.' }
} else {
  for (const [name, plugin] of Object.entries(selectedPlugins)) {
    const rows = []
    for (const task of selectedTasks) {
      fixture.posted.length = 0
      const session = await openBotBrowser({ botId: `${name}-${task.id}`.replace(/[^a-z0-9-]/gi, '-'), profileRoot, headless: true, playwrightUrl })
      console.log(`run ${name} ${task.id}`)
      const row = await runModelTask(plugin, task, session, fixture)
      console.log(`  ${row.ok ? 'ok' : 'fail'} steps=${row.steps} ${row.error ?? row.answer.slice(0, 80)}`)
      rows.push(row)
      await session.close()
    }
    results.plugins[name] = { rows }
  }
}

await fixture.close()
await mkdir(outDir, { recursive: true })
await writeFile(new URL('./results.json', outDir), JSON.stringify(results, null, 2))
console.log(JSON.stringify({ key: results.key, driver: results.driver, isolation: results.isolation }, null, 2))