Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

21.8 KB
Raw Blame
import assert from 'node:assert/strict'
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { test } from 'node:test'
import { createRuntime } from '../index.js'
import { TEAM_TOOLS } from '../src/host/constants.js'
import { fakeContext } from './fake-ctx.js'

// tools.js keeps the DSH names a Bot Session never serves in one place; the
// assemble must add none of them.
const HIDDEN = ['ask_user_question', 'subagent', 'list_subagent_models', 'send_message']

// A Bot-owned Session gets its Agent before ds-bot records the owner, so the
// scoped show_view and the hidden-tool restrict used to wait for a later
// resolveAgent that a held kickoff never ran. These tests drive each creation
// path with another plugin's global show_view already in place and check the
// Agent's own tool list before its first step.
const TEAM = {
  version: 1,
  mainBotIds: ['a'],
  workspaceId: 'w1',
  bots: {
    a: { id: 'a', name: 'Chief', role: 'Boss', sessionId: 'a', createdAt: 1 },
    b: { id: 'b', name: 'Writer', role: 'Write', sessionId: 'b', createdAt: 2 },
  },
  rooms: {
    r1: { id: 'r1', name: 'Ops', members: ['a', 'b'], mode: 'admin', admin: 'a', named: true, sessions: {}, log: [], seq: 0, seen: {} },
  },
  agents: {},
  exchanges: [],
  exchangeTurns: {},
  questions: {},
  prefs: {},
  revision: 0,
}

let rt
const start = (team = TEAM, services = {}, globals = [], shared = {}) => {
  const home = shared.home ?? mkdtempSync(join(tmpdir(), 'ds-bot-scoped-'))
  if (shared.home === undefined) writeFileSync(join(home, 'state.json'), JSON.stringify(team))
  // `scoped` and `restricted` model each Session's own tool layer, and
  // `liveAgents` the DSH registry: shared across two runtimes they keep a
  // Session's Agent alive through a disable and re-enable.
  const scoped = shared.scoped ?? []
  const restricted = shared.restricted ?? []
  const liveAgents = shared.liveAgents ?? new Map()
  services.agents ??= { list: () => [...liveAgents.values()], get: id => liveAgents.get(id) }
  const ctx = fakeContext({ services })
  // Another plugin's global show_view, in place before ds-bot loads.
  ctx.tools.register({ name: 'show_view', description: 'global', execute: async () => 'global' })
  for (const tool of globals) ctx.tools.register(tool)
  // DSH refuses a second global registration of one name — the throw that killed
  // ds-bot's boot in #703 — so the fake refuses the same way.
  const plainRegister = ctx.tools.register.bind(ctx.tools)
  ctx.tools.register = (tool) => {
    if (ctx.registeredTools.has(tool.name)) throw new Error(`tool "${tool.name}" is already registered`)
    return plainRegister(tool)
  }
  let sessions = 0
  const agentOf = (sessionId) => {
    const held = liveAgents.get(sessionId)
    if (held !== undefined) return held
    const nextStep = []
    const nextTurn = []
    const agent = {
      id: sessionId,
      session: { id: sessionId, header: { cwd: '/team' }, requestHeader: () => undefined, deriveMessages: () => [] },
      ctx: {
        tools: {
          register: (tool) => {
            const entry = { sessionId, name: tool.name, tool }
            scoped.push(entry)
            // DSH's disposer unregisters; the fake drops the record.
            return () => { const at = scoped.indexOf(entry); if (at !== -1) scoped.splice(at, 1) }
          },
          restrict: (filter) => {
            const entry = { sessionId, deny: filter.deny ?? [] }
            restricted.push(entry)
            return () => { const at = restricted.indexOf(entry); if (at !== -1) restricted.splice(at, 1) }
          },
        },
        effect: execute => (typeof execute === 'function' ? execute() : undefined),
        on: () => () => {},
      },
      status: 'idle',
      followup() {},
      steer() {},
      inject() {},
      send(message, target) {
        ;(target === 'next-turn' ? nextTurn : nextStep).push(message)
      },
      cancel() {},
      async whenIdle() {},
      async runMaintenance(fn) { return fn(AbortSignal.timeout(60_000)) },
      inbox: {
        nextStep,
        nextTurn,
        prepend(target, message) { ;(target === 'next-turn' ? nextTurn : nextStep).unshift(message) },
        append(target, message) { ;(target === 'next-turn' ? nextTurn : nextStep).push(message) },
        remove(id) {
          for (const list of [nextStep, nextTurn]) {
            const at = list.findIndex(message => message.id === id)
            if (at !== -1) { list.splice(at, 1); return true }
          }
          return false
        },
      },
    }
    liveAgents.set(sessionId, agent)
    return agent
  }
  // ask_user_question is served, so the deny list is not empty in a Bot scope;
  // a Session's own registrations join its list and shadow globals of one name,
  // as DSH's scope view does.
  ctx.tools.schemas = (agent) => {
    const view = new Map()
    for (const name of ['ask_user_question', 'show_view']) view.set(name, { name })
    for (const entry of scoped.filter(entry => entry.sessionId === agent?.id)) {
      view.set(entry.name, { name: entry.name, description: entry.tool.description, parameters: entry.tool.parameters })
    }
    return [...view.values()]
  }
  Object.assign(ctx, {
    llm: { listProviders: () => [], listModels: async () => [], resolveModelInfo: async () => ({}) },
    workspaceRegistry: {
      get: id => (id === 'w1' ? { id: 'w1', path: '/team' } : undefined),
      list: () => [{ id: 'w1', path: '/team' }],
      resolveByPath: async () => undefined,
      create: async path => ({ id: 'w1', path }),
      archiveSession: async () => {},
    },
    sessionController: {
      create: async () => ({ sessionId: `s${++sessions}` }),
      resolveAgent: async sessionId => ({ agent: agentOf(sessionId) }),
      selectModel: async () => {},
      page: async () => ({ records: [], hasMore: false }),
    },
    sessionTitle: { rename: async () => {} },
    sessions: { flush: async () => {} },
  })
  // Sessions named in shared.live get their Agent into the registry before the
  // runtime and its first load — the state a restart leaves behind (#826).
  for (const liveId of shared.live ?? []) agentOf(liveId)
  let route
  const register = ctx.connection.fetch.register
  ctx.connection.fetch.register = (options) => { route = options; return register(options) }
  rt = createRuntime(ctx, { home })
  const call = async (endpoint, payload) => (await route.fetch(new Request('http://localhost/api/bot', {
    method: 'POST', body: JSON.stringify({ endpoint, payload }),
  }))).json()
  const emit = (id, type, data) => {
    for (const listener of ctx.listeners.get('session/event') ?? []) {
      if (listener.length >= 2) listener({ id }, { type, data })
    }
  }
  const agentCreated = async (sessionId) => {
    for (const listener of ctx.listeners.get('agent/created') ?? []) await listener({ agent: agentOf(sessionId) })
  }
  const tool = (name, args, sessionId, exec = {}) =>
    rt.teamTools.find(d => d.name === name).execute(args, { agent: agentOf(sessionId), name, callId: 'c1', ...exec })
  const unload = () => ctx.dispose()
  const stop = async () => {
    await unload()
    rmSync(home, { recursive: true, force: true, maxRetries: 5 })
  }
  return { ctx, call, tool, emit, agentCreated, agentOf, scoped, restricted, home, unload, stop }
}

const until = async (check, ms = 3000) => {
  const end = Date.now() + ms
  while (!check()) {
    if (Date.now() > end) throw new Error('until: the condition did not hold')
    await new Promise(resolve => setTimeout(resolve, 20))
  }
}

const hasShowView = (host, id) => host.scoped.some(entry => entry.sessionId === id && entry.name === 'show_view')
const denied = (host, id) => host.restricted.some(entry => entry.sessionId === id && entry.deny.includes('ask_user_question'))
const scopedNames = (host, id) => host.scoped.filter(entry => entry.sessionId === id).map(entry => entry.name)

test('a Bot Session gets every team tool in its own scope', async (t) => {
  const host = start()
  t.after(host.stop)
  await host.call('state', {})
  await host.agentCreated('a')
  assert.deepEqual(scopedNames(host, 'a').sort(), [...TEAM_TOOLS].filter(name => name !== 'read_browser').sort())
})

test('a Bot moved onto a local agent route gets the approval bridge, and loses it moving off', async (t) => {
  const host = start()
  t.after(host.stop)
  await host.call('state', {})
  await host.agentCreated('b')
  assert.equal(scopedNames(host, 'b').includes('approve_agent_action'), false)
  rt.isLocalAgentRef = ref => typeof ref === 'string' && ref.startsWith('acp-')
  const bot = rt.botOf('b')
  bot.model = 'acp-fake/default'
  rt.hideBotUnusedTools(host.agentOf('b'))
  assert.equal(scopedNames(host, 'b').includes('approve_agent_action'), true)
  assert.equal(scopedNames(host, 'b').filter(name => name === 'message_bot').length, 1)
  bot.model = 'deepseek-official/deepseek-flash'
  rt.hideBotUnusedTools(host.agentOf('b'))
  assert.equal(scopedNames(host, 'b').includes('approve_agent_action'), false)
  assert.equal(scopedNames(host, 'b').includes('message_bot'), true)
})

test('a Session ds-bot does not own sees none of the team tools', async (t) => {
  const host = start()
  t.after(host.stop)
  await host.call('state', {})
  await host.agentCreated('plain1')
  assert.deepEqual(scopedNames(host, 'plain1'), [])
  // Nothing of the team's reaches the global layer a plain session reads; the
  // only team-named global is another plugin's show_view.
  const leaked = [...host.ctx.registeredTools.keys()].filter(name => TEAM_TOOLS.includes(name))
  assert.deepEqual(leaked, ['show_view'])
  assert.equal(host.ctx.registeredTools.get('show_view').description, 'global')
})

test("a group's own Session is scoped to group_relay alone", async (t) => {
  const host = start()
  t.after(host.stop)
  await host.call('state', {})
  await host.agentCreated('r1')
  assert.deepEqual(scopedNames(host, 'r1'), ['group_relay'])
})

test("another plugin's global recall cannot stop ds-bot or shadow its own", async (t) => {
  // #703: a same-named global tool from another plugin must not fail apply(),
  // and inside a Bot Session ds-bot's recall still wins.
  const host = start(TEAM, {}, [{ name: 'recall', description: 'foreign recall', execute: async () => 'foreign' }])
  t.after(host.stop)
  assert.equal(host.ctx.registeredTools.get('recall').description, 'foreign recall')
  await host.call('state', {})
  await host.agentCreated('a')
  const recall = host.scoped.find(entry => entry.sessionId === 'a' && entry.name === 'recall')
  assert.match(String(recall?.tool?.description), /^Search your memory/)
})

test('a Bot made without a model has the scoped tools before its first step', async (t) => {
  const host = start({ ...TEAM, rooms: {} })
  t.after(host.stop)
  const created = await host.call('create-bot', { name: 'Solo' })
  assert.equal(created.ok, true)
  const id = created.value.id
  assert.ok(hasShowView(host, id), 'the Bot scope has ds-bot\'s show_view over the global one')
  assert.ok(denied(host, id), 'the blocking question tool is denied')
})

test('a group Session made for a member has the scoped tools before its first step', async (t) => {
  const host = start()
  t.after(host.stop)
  await host.call('state', {})
  const stop = new AbortController()
  t.after(() => stop.abort())
  // The user's post starts a round; Writer has no group Session yet, so the round
  // makes one — which used to keep whatever tools a global registration left.
  host.emit('r1', 'user/message', { content: [{ type: 'text', text: 'launch moved' }], source: { kind: 'user' } })
  void host.tool('group_relay', {}, 'r1', { callId: 'r1', signal: stop.signal }).catch(() => {})
  await until(() => hasShowView(host, 's1'), 5000)
  assert.ok(denied(host, 's1'), 'the blocking question tool is denied in the group scope')
})

test('a fresh part Session has the scoped tools before its first step', async (t) => {
  const project = mkdtempSync(join(tmpdir(), 'ds-bot-scoped-proj-'))
  const host = start({ ...TEAM, rooms: {} }, {
    sandboxPolicy: { resolve: () => ({ mode: 'workspace-write' }) },
    approval: { request: async () => 'allowed-once' },
  })
  t.after(async () => {
    await host.stop()
    rmSync(project, { recursive: true, force: true, maxRetries: 5 })
  })
  await host.call('state', {})
  host.agentOf('b').status = 'running'
  const moved = await host.tool('update_bot', { work_dir: project }, 'b')
  assert.equal(String(moved).includes('From your next step on'), true)
  // The next step sees the folder change and switches to a fresh part.
  const preStep = host.ctx.listeners.get('agent/pre-step').at(-1)
  const answer = await preStep(
    { agent: host.agentOf('b'), messages: [{ role: 'user', content: [{ type: 'text', text: 'go on' }], source: { kind: 'user' } }], step: 2, signal: AbortSignal.timeout(60_000) },
    async () => 'next',
  )
  assert.deepEqual(answer, { kind: 'reject' })
  await until(() => hasShowView(host, 's1'), 5000)
  assert.ok(denied(host, 's1'), 'the blocking question tool is denied in the fresh part')
})

test('a Session ds-bot does not own keeps only the global show_view', async (t) => {
  const host = start()
  t.after(host.stop)
  await host.call('state', {})
  await host.agentCreated('plain1')
  assert.ok(!hasShowView(host, 'plain1'), 'no scoped show_view shadows the global one')
  assert.ok(!host.restricted.some(entry => entry.sessionId === 'plain1'), 'no restrict applies outside the team')
})

test("a deleted Bot's Session keeps none of the scope when it is adopted as a plain one", async (t) => {
  const host = start()
  t.after(host.stop)
  await host.call('state', {})
  await host.agentCreated('b')
  assert.ok(hasShowView(host, 'b'))
  assert.ok(denied(host, 'b'))
  assert.equal((await host.call('delete-bot', { id: 'b' })).ok, true)
  assert.deepEqual(scopedNames(host, 'b'), [])
  assert.equal(host.restricted.some(entry => entry.sessionId === 'b'), false)
  // The Session lives on unowned: adopting it into the DSH Agent must not bring
  // the team tools back.
  const agent = (await host.call('add-agent', {})).value
  assert.equal((await host.call('agent-adopt', { agentId: agent.id, sessionId: 'b' })).ok, true)
  assert.deepEqual(scopedNames(host, 'b'), [])
  assert.equal(host.restricted.some(entry => entry.sessionId === 'b'), false)
})

test('a member who leaves the group loses the scoped tools', async (t) => {
  const team = {
    ...TEAM,
    bots: { ...TEAM.bots, c: { id: 'c', name: 'Scout', role: 'Scout', sessionId: 'c', createdAt: 3 } },
    rooms: { r1: { ...TEAM.rooms.r1, members: ['a', 'b', 'c'], sessions: { b: 'sb' } } },
  }
  const host = start(team)
  t.after(host.stop)
  await host.call('state', {})
  await host.agentCreated('sb')
  assert.ok(hasShowView(host, 'sb'))
  assert.ok(denied(host, 'sb'))
  assert.equal((await host.call('update-room', { id: 'r1', remove: ['b'] })).ok, true)
  assert.deepEqual(scopedNames(host, 'sb'), [])
  assert.equal(host.restricted.some(entry => entry.sessionId === 'sb'), false)
  // Resolving the archived Session again does not bring the tools back.
  await host.agentCreated('sb')
  assert.deepEqual(scopedNames(host, 'sb'), [])
})

test('a deleted group loses its relay scope', async (t) => {
  const host = start()
  t.after(host.stop)
  await host.call('state', {})
  await host.agentCreated('r1')
  assert.deepEqual(scopedNames(host, 'r1'), ['group_relay'])
  assert.equal((await host.call('delete-room', { id: 'r1' })).ok, true)
  assert.deepEqual(scopedNames(host, 'r1'), [])
})

test('a Session that becomes a Bot\'s again gets the scope back', async (t) => {
  const host = start()
  t.after(host.stop)
  await host.call('state', {})
  await host.agentCreated('b')
  assert.ok(hasShowView(host, 'b'))
  assert.equal((await host.call('delete-bot', { id: 'b' })).ok, true)
  assert.deepEqual(scopedNames(host, 'b'), [])
  // A Bot record naming the Session again — here written directly, the way an
  // adopted state could carry it — serves the tools again.
  rt.state.bots.b = { id: 'b', name: 'Writer', role: 'Write', sessionId: 'b', createdAt: 2 }
  await rt.save()
  await host.agentCreated('b')
  assert.ok(hasShowView(host, 'b'))
})

// #826: after a DSH restart the creation path can miss a live Agent and nothing
// scoped it again, so the Session ran its requests without the team tools. The
// step's assemble scopes an owned Session the creation path missed and adds the
// names the request list was already built without.
const assemble = (host, agent, assembly = { tools: [], sections: [] }) => {
  const listener = host.ctx.listeners.get('system-prompt/assemble')
    .find(listener => /scopeTeamTools/.test(String(listener)))
  return listener(assembly, { agent, signal: AbortSignal.timeout(60_000) }, async () => assembly)
}

test("a group Session the creation path missed is scoped at the step's assemble", async (t) => {
  const host = start()
  t.after(host.stop)
  await host.call('state', {})
  // The Agent is live but agent/created never ran for it and no save re-synced
  // the scopes — the state a restart left the reporter's group in.
  const agent = host.agentOf('r1')
  const assembled = await assemble(host, agent, { tools: [{ name: 'read' }], sections: [] })
  assert.deepEqual(scopedNames(host, 'r1'), ['group_relay'])
  assert.deepEqual(assembled.tools.map(tool => tool.name), ['read', 'group_relay'])
})

test("a Bot's own Session the creation path missed gets the whole scope at the assemble", async (t) => {
  const host = start()
  t.after(host.stop)
  await host.call('state', {})
  const agent = host.agentOf('a')
  const assembled = await assemble(host, agent, { tools: [{ name: 'read' }], sections: [] })
  const expected = [...TEAM_TOOLS].filter(name => name !== 'read_browser').sort()
  assert.deepEqual(scopedNames(host, 'a').sort(), expected)
  assert.deepEqual(assembled.tools.map(tool => tool.name).sort(), ['read', ...expected].sort())
  // The hidden DSH tools are masked and filtered as on the creation path: none
  // of their names joined the list the assemble added.
  assert.ok(denied(host, 'a'), 'the blocking question tool is denied')
  assert.ok(!HIDDEN.some(name => assembled.tools.some(tool => tool.name === name)))
})

test('the assemble registers nothing twice in an already scoped Session', async (t) => {
  const host = start()
  t.after(host.stop)
  await host.call('state', {})
  await host.agentCreated('a')
  const before = host.scoped.length
  const assembly = { tools: [{ name: 'read' }], sections: [] }
  assert.equal(await assemble(host, host.agentOf('a'), assembly), assembly)
  assert.equal(host.scoped.length, before)
})

test('the assemble passes an unowned Session through unchanged', async (t) => {
  const host = start()
  t.after(host.stop)
  await host.call('state', {})
  const assembly = { tools: [{ name: 'read' }], sections: [] }
  assert.equal(await assemble(host, host.agentOf('plain1'), assembly), assembly)
  assert.deepEqual(scopedNames(host, 'plain1'), [])
})

test('a Session the load itself scoped still gets its tools into the step\'s list', async (t) => {
  // The restart case of #826: the Agent was already live when the runtime's
  // first load scoped it — the names were never on a collected tool list, so
  // the assemble must still add them to this step's.
  const host = start(TEAM, {}, [], { live: ['r1'] })
  t.after(host.stop)
  const assembled = await assemble(host, host.agentOf('r1'), { tools: [{ name: 'read' }], sections: [] })
  assert.deepEqual(scopedNames(host, 'r1'), ['group_relay'])
  assert.deepEqual(assembled.tools.map(tool => tool.name), ['read', 'group_relay'])
})

test('a PTC group Session gets its relay scoped but keeps run_code alone in the list', async (t) => {
  // A group made under the PTC preset relays through run_code; adding the scoped
  // group_relay to the list would make the hook pick the call it cannot declare.
  const host = start()
  t.after(host.stop)
  const agent = host.agentOf('r1')
  const assembly = { tools: [{ name: 'run_code' }], sections: [] }
  assert.equal(await assemble(host, agent, assembly), assembly)
  assert.deepEqual(scopedNames(host, 'r1'), ['group_relay'])
})

test('a same-named global in the list is replaced by the scoped schema', async (t) => {
  // Another plugin's global show_view was collected for the request; the scoped
  // one that shadows it at execution replaces its schema in the list.
  const host = start()
  t.after(host.stop)
  const agent = host.agentOf('a')
  const assembled = await assemble(host, agent, { tools: [{ name: 'show_view', description: 'global' }], sections: [] })
  const named = assembled.tools.filter(tool => tool.name === 'show_view')
  assert.equal(named.length, 1)
  assert.notEqual(named[0].description, 'global')
})

test('the assemble adds the recovered names once, not on every step', async (t) => {
  const host = start()
  t.after(host.stop)
  const agent = host.agentOf('r1')
  const first = await assemble(host, agent, { tools: [{ name: 'read' }], sections: [] })
  assert.deepEqual(first.tools.map(tool => tool.name), ['read', 'group_relay'])
  const second = { tools: [{ name: 'read' }], sections: [] }
  assert.equal(await assemble(host, agent, second), second)
})

test('a live Bot Session gets its team tools back after a disable and re-enable', async (t) => {
  // A hot reload lifts the scope of Agents that are still live, and
  // agent/created does not fire again: the sync at the next load must scope
  // them before their next request.
  const shared = { scoped: [], restricted: [], liveAgents: new Map() }
  const first = start(TEAM, {}, [], shared)
  shared.home = first.home
  t.after(first.unload)
  await first.call('state', {})
  await first.agentCreated('a')
  assert.ok(hasShowView(first, 'a'))
  assert.ok(denied(first, 'a'))
  await first.unload()
  assert.deepEqual(scopedNames(first, 'a'), [])
  assert.equal(denied(first, 'a'), false)
  const second = start(TEAM, {}, [], shared)
  t.after(second.stop)
  await second.call('state', {})
  assert.ok(hasShowView(second, 'a'), 'the scoped team tools are back')
  assert.ok(denied(second, 'a'), 'the blocking question tool is denied again')
})