A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { install } from '../src/host/models.js'
const PROVIDERS = {
deepseek: [{ id: 'deepseek-v4.1-flash', name: 'DeepSeek V4.1 Flash' }, { id: 'deepseek-v4-pro' }],
ollama: [{ id: 'kimi-k3' }, { id: 'deepseek-v4-pro' }],
}
function runtime(defaultSelection) {
const ctx = {
llm: {
listProviders: () => Object.keys(PROVIDERS).map(id => ({ id })),
listModels: async id => PROVIDERS[id],
},
get: name => (name === 'agentDefaultModel' && defaultSelection ? { currentSelection: () => defaultSelection } : undefined),
}
const rt = { ctx, state: { bots: {} } }
install(rt)
return rt
}
test('the default model is the DSH default when served, else the first served model', async () => {
assert.equal(await runtime({ provider: 'ollama', model: 'kimi-k3' }).defaultModel(), 'ollama/kimi-k3')
assert.equal(await runtime({ provider: 'gone', model: 'x' }).defaultModel(), 'deepseek/deepseek-v4.1-flash')
assert.equal(await runtime(undefined).defaultModel(), 'deepseek/deepseek-v4.1-flash')
})
// DSH serves the models of a provider whose key is not set; every turn on them fails.
function withoutKeys(defaultSelection, providers) {
const rt = runtime(defaultSelection)
rt.missingKeys = async () => new Set(providers)
return rt
}
test('a model that needs a key is never a default or a fallback, though a Bot keeps its own', async () => {
const rt = withoutKeys({ provider: 'deepseek', model: 'deepseek-v4-pro' }, ['deepseek'])
assert.equal(await rt.defaultModel(), 'ollama/kimi-k3')
assert.equal(rt.cachedDefault(), 'ollama/kimi-k3')
assert.equal(rt.isUsable('ollama/kimi-k3'), true)
assert.equal(rt.isUsable('deepseek/deepseek-v4-pro'), false)
assert.equal(rt.isUsable(undefined), false)
assert.equal(await rt.desiredModel({ model: 'deepseek/deepseek-v4-pro' }), 'deepseek/deepseek-v4-pro')
assert.equal(await rt.desiredModel({ model: 'gone/model', appliedModel: 'deepseek/deepseek-v4-pro' }), 'ollama/kimi-k3')
assert.equal(await rt.desiredModel({ model: 'gone/model', appliedModel: 'ollama/deepseek-v4-pro' }), 'ollama/deepseek-v4-pro')
})
test('with every model needing a key, nothing is picked', async () => {
const rt = withoutKeys({ provider: 'ollama', model: 'kimi-k3' }, ['deepseek', 'ollama'])
assert.equal(await rt.defaultModel(), undefined)
assert.equal(rt.cachedDefault(), undefined)
assert.equal(await rt.desiredModel({ model: 'gone/model', appliedModel: 'ollama/kimi-k3' }), undefined)
})
test('a model is named by its ref, or by an id or name only one provider serves', async () => {
const rt = runtime(undefined)
assert.equal(await rt.resolveModel('ollama/kimi-k3'), 'ollama/kimi-k3')
assert.equal(await rt.resolveModel(' Kimi-K3 '), 'ollama/kimi-k3')
assert.equal(await rt.resolveModel('DeepSeek V4.1 Flash'), 'deepseek/deepseek-v4.1-flash')
await assert.rejects(rt.resolveModel('deepseek-v4-pro'), /more than one provider.*deepseek\/deepseek-v4-pro, ollama\/deepseek-v4-pro/)
await assert.rejects(rt.resolveModel('kimi'), /not available\. Available models: deepseek\/deepseek-v4\.1-flash/)
})
test('a Bot runs on its own model, else the one it last ran on, else the default', async () => {
const rt = runtime({ provider: 'ollama', model: 'kimi-k3' })
assert.equal(await rt.desiredModel({ model: 'deepseek/deepseek-v4-pro', appliedModel: 'ollama/kimi-k3' }), 'deepseek/deepseek-v4-pro')
assert.equal(await rt.desiredModel({ model: 'gone/model', appliedModel: 'deepseek/deepseek-v4-pro' }), 'deepseek/deepseek-v4-pro')
assert.equal(await rt.desiredModel({ model: 'gone/model' }), 'ollama/kimi-k3')
})
test('choosing a Bot model leaves the user\'s DSH default model as it was', async () => {
const writes = []
let current = { provider: 'ollama', model: 'kimi-k3' }
const defaults = {
currentSelection: () => ({ ...current }),
saveSelection: async (next) => { writes.push(`${next.provider}/${next.model}`); current = { ...next } },
}
const ctx = {
llm: { listProviders: () => [], listModels: async () => [] },
get: name => (name === 'agentDefaultModel' ? defaults : undefined),
// Like DSH: the selection is also saved as the default, in the background.
sessionController: { selectModel: async ({ provider, model }) => { void defaults.saveSelection({ provider, model }) } },
}
const rt = { ctx, state: { bots: {} }, warn() {} }
install(rt)
await Promise.all([
rt.selectSessionModel('s1', { provider: 'deepseek', model: 'deepseek-v4-pro' }),
rt.selectSessionModel('s2', { provider: 'deepseek', model: 'deepseek-v4.1-flash' }),
rt.selectSessionModel('s3', { provider: 'ollama', model: 'kimi-k3' }),
])
assert.deepEqual(current, { provider: 'ollama', model: 'kimi-k3' })
assert.deepEqual(writes, ['deepseek/deepseek-v4-pro', 'ollama/kimi-k3', 'deepseek/deepseek-v4.1-flash', 'ollama/kimi-k3', 'ollama/kimi-k3'])
})
test('a Bot with no model yet changes nothing until the user picks one', async () => {
const rt = runtime({ provider: 'ollama', model: 'kimi-k3' })
assert.equal(await rt.desiredModel({ appliedModel: 'deepseek/deepseek-v4-pro' }), undefined)
assert.equal(await rt.desiredModel({}), undefined)
await rt.catalog()
assert.equal(rt.cachedDefault(), 'ollama/kimi-k3')
})
// A runtime with two providers, a DSH default, and a session controller that, like
// DSH, saves every Session selection as the default.
function bench({ providers = { deepseek: ['v4-flash'], stepfun: ['step-5'] }, fallback = { provider: 'stepfun', model: 'step-5' }, reasoning = {}, services = {} } = {}) {
let current = { ...fallback }
const selected = []
const saved = []
const listeners = new Map()
const ctx = {
on(name, listener) { listeners.set(name, listener); return () => listeners.delete(name) },
get: name => (name === 'agentDefaultModel'
? { currentSelection: () => ({ ...current }), saveSelection: async (next) => { saved.push(next); current = { ...next } } }
: services[name]),
llm: {
listProviders: () => Object.keys(providers).map(id => ({ id, name: id.toUpperCase() })),
listModels: async id => providers[id].map(model => ({ id: model, name: model, inputModalities: ['text'] })),
resolveModelInfo: async (provider, model) => reasoning[`${provider}/${model}`] ?? {},
},
sessionController: {
selectModel: async ({ sessionId, provider, model, reasoningEffort }) => {
selected.push(`${sessionId}:${provider}/${model}${reasoningEffort === undefined ? '' : `:${reasoningEffort}`}`)
current = { provider, model }
},
},
}
const rt = { ctx, state: { bots: {} }, chatOf: id => id, groupSessionsOf: () => [], save: async () => {}, warn: () => {} }
install(rt)
return { rt, selected, saved, providers, listeners, current: () => current }
}
test('the catalog is every model DSH serves, with no lab', async () => {
const { rt } = bench()
assert.deepEqual((await rt.catalog()).map(entry => [entry.ref, entry.providerName, 'lab' in entry]), [
['deepseek/v4-flash', 'DEEPSEEK', false],
['stepfun/step-5', 'STEPFUN', false],
])
})
test('a fresh catalog sees a model list changed in DSH settings at once', async () => {
const { rt, providers } = bench()
await rt.catalog()
providers.deepseek = ['deepseek-flash', 'deepseek-v4-pro']
assert.ok((await rt.catalog()).some(entry => entry.ref === 'deepseek/v4-flash'))
assert.deepEqual((await rt.catalog({ fresh: true })).filter(entry => entry.provider === 'deepseek').map(entry => entry.id), ['deepseek-flash', 'deepseek-v4-pro'])
})
test('applying a Bot model leaves the DSH default as the user set it', async () => {
const { rt, selected, current } = bench()
const bot = { id: 'b1', model: 'deepseek/v4-flash' }
assert.equal(await rt.applyModel(bot), 'deepseek/v4-flash')
assert.deepEqual(selected, ['b1:deepseek/v4-flash'])
assert.deepEqual(current(), { provider: 'stepfun', model: 'step-5' })
assert.equal(bot.appliedModel, 'deepseek/v4-flash')
})
test('a Bot model applies its thinking level, and only re-selects on a change', async () => {
const { rt, selected } = bench({ reasoning: { 'deepseek/v4-flash': { reasoning: { efforts: [{ id: 'low', name: 'Low' }, { id: 'high', name: 'High' }], defaultEffort: 'high' } } } })
const bot = { id: 'b1', model: 'deepseek/v4-flash', effort: 'low' }
await rt.applyModel(bot)
assert.deepEqual(selected, ['b1:deepseek/v4-flash:low'])
assert.equal(bot.appliedEffort, 'low')
await rt.applyModel(bot)
assert.equal(selected.length, 1)
// A level the model does not serve is never sent: the model's default applies.
bot.effort = 'max'
await rt.applyModel(bot)
assert.deepEqual(selected.at(-1), 'b1:deepseek/v4-flash')
assert.equal(bot.appliedEffort, undefined)
})
test('group selections sync per Session even when the Bot-level fields already match', async () => {
const { rt, selected } = bench({ reasoning: { 'deepseek/v4-flash': { reasoning: { efforts: [{ id: 'low' }, { id: 'high' }], defaultEffort: 'high' } } } })
const bot = { id: 'b1', model: 'deepseek/v4-flash', effort: 'low', appliedModel: 'deepseek/v4-flash', appliedEffort: 'low' }
const groups = ['g1', 'g2']
rt.groupSessionsOf = () => groups
const select = rt.ctx.sessionController.selectModel
rt.ctx.sessionController.selectModel = async (request) => {
await select(request)
return { selected: { provider: request.provider, model: request.model, reasoningEffort: request.reasoningEffort ?? 'high' } }
}
await rt.applyModel(bot)
assert.deepEqual(selected, ['g1:deepseek/v4-flash:low', 'g2:deepseek/v4-flash:low'])
await rt.applyModel(bot)
assert.equal(selected.length, 2)
groups.push('g3')
await rt.applyModel(bot)
assert.deepEqual(selected.slice(2), ['g3:deepseek/v4-flash:low'])
delete bot.effort
await rt.applyModel(bot)
assert.deepEqual(selected.slice(3), ['b1:deepseek/v4-flash', 'g1:deepseek/v4-flash', 'g2:deepseek/v4-flash', 'g3:deepseek/v4-flash'])
// The provider's resolved default is not an explicit Bot choice to reapply.
await rt.applyModel(bot)
assert.equal(selected.length, 7)
await rt.applyModel(bot, { force: true })
assert.equal(selected.length, 11)
})
test('a failed group selection is retried without reselecting the own chat or successful groups', async () => {
const { rt, selected } = bench()
const bot = { id: 'b1', model: 'deepseek/v4-flash', appliedModel: 'deepseek/v4-flash' }
rt.groupSessionsOf = () => ['g1', 'g2']
const select = rt.ctx.sessionController.selectModel
let failed = false
rt.ctx.sessionController.selectModel = async (request) => {
if (request.sessionId === 'g2' && !failed) {
failed = true
throw new Error('Session unavailable')
}
return select(request)
}
await rt.applyModel(bot)
assert.deepEqual(selected, ['g1:deepseek/v4-flash'])
await rt.applyModel(bot)
assert.deepEqual(selected, ['g1:deepseek/v4-flash', 'g2:deepseek/v4-flash'])
})
test('resolveEffort takes an id or a name, and refuses what the model has not', async () => {
const { rt } = bench({ reasoning: {
'deepseek/v4-flash': { reasoning: { efforts: [{ id: 'off', name: 'Off' }, { id: 'high', name: 'High' }] } },
'stepfun/step-5': {},
} })
assert.equal(await rt.resolveEffort('deepseek/v4-flash', 'high'), 'high')
assert.equal(await rt.resolveEffort('deepseek/v4-flash', ' Off '), 'off')
assert.equal(await rt.resolveEffort('deepseek/v4-flash', ''), undefined)
await assert.rejects(rt.resolveEffort('stepfun/step-5', 'high'), /no thinking levels/)
await assert.rejects(rt.resolveEffort('deepseek/v4-flash', 'max'), /not available for deepseek\/v4-flash\. Available: Off, High/)
})
test('the catalog attaches thinking levels to models a Bot uses', async () => {
const { rt } = bench({ reasoning: { 'deepseek/v4-flash': { reasoning: { efforts: [{ id: 'high', name: 'High' }], defaultEffort: 'high' } } } })
rt.state.bots.b1 = { id: 'b1', model: 'deepseek/v4-flash' }
const entries = await rt.catalog()
assert.deepEqual(entries.find(entry => entry.ref === 'deepseek/v4-flash')?.reasoning, { efforts: [{ id: 'high', name: 'High' }], defaultEffort: 'high' })
assert.equal(entries.find(entry => entry.ref === 'stepfun/step-5')?.reasoning, undefined)
})
// #637: a hosted loop engine publishes a route that resolves models but refuses every
// generation with HOSTED_ENGINE_ROUTE; once it refuses, nothing chooses it again.
test('a route refused for not being a model endpoint is chosen nowhere again', async () => {
const { rt, selected } = bench({ providers: { external: ['default'], deepseek: ['v4-flash'] }, fallback: { provider: 'external', model: 'default' } })
const pinned = { id: 'b1', model: 'external/default', appliedModel: 'external/default' }
const clean = { id: 'b2', model: 'deepseek/v4-flash' }
rt.state.bots = { b1: pinned, b2: clean }
await rt.routeRefused('external/default', pinned)
// The pinned Bot moves to what it would be given today and re-selects its Session.
assert.equal(pinned.model, 'deepseek/v4-flash')
assert.equal(pinned.appliedModel, 'deepseek/v4-flash')
assert.equal(clean.model, 'deepseek/v4-flash')
assert.deepEqual(selected, ['b1:deepseek/v4-flash'])
// Pickers, the deployment default and explicit naming keep it out too.
assert.deepEqual((await rt.catalog({ fresh: true })).map(entry => entry.ref), ['deepseek/v4-flash'])
assert.deepEqual(rt.cachedCatalog().map(entry => entry.ref), ['deepseek/v4-flash'])
assert.equal(await rt.defaultModel(), 'deepseek/v4-flash')
await assert.rejects(rt.resolveModel('external/default'), /Model external\/default is not available/)
})
test('a Bot on a clean record but a poisoned Session re-selects its own model', async () => {
const { rt, selected } = bench({ providers: { external: ['default'], deepseek: ['v4-flash'] }, fallback: { provider: 'external', model: 'default' } })
// Its group Session ran on another Bot's route; its own record never named it.
const bot = { id: 'b1', model: 'deepseek/v4-flash', appliedModel: 'deepseek/v4-flash' }
rt.state.bots = { b1: bot }
await rt.routeRefused('external/default', bot)
assert.equal(bot.model, 'deepseek/v4-flash')
assert.deepEqual(selected, ['b1:deepseek/v4-flash'])
})
// Every Bot the route was pinned to may have its Session still on it, even one whose
// record already names the replacement; re-pinning the record alone leaves that
// Session failing on every turn.
test('every Bot re-pinned off a refused route re-selects its Session', async () => {
const { rt, selected } = bench({ providers: { external: ['default'], deepseek: ['v4-flash'] }, fallback: { provider: 'external', model: 'default' } })
const failed = { id: 'b1', model: 'external/default', appliedModel: 'external/default' }
const alsoPinned = { id: 'b2', model: 'external/default', appliedModel: 'deepseek/v4-flash' }
rt.state.bots = { b1: failed, b2: alsoPinned }
await rt.routeRefused('external/default', failed)
assert.equal(alsoPinned.model, 'deepseek/v4-flash')
assert.equal(alsoPinned.appliedModel, 'deepseek/v4-flash')
assert.deepEqual([...selected].sort(), ['b1:deepseek/v4-flash', 'b2:deepseek/v4-flash'])
})
// The sync key must see the filtered catalog: the catalog itself does not change
// on a refusal, so an unfiltered key would never run the sync again.
test('a refusal changes the sync key, so the next sync applies what the records say', async () => {
const { rt, selected } = bench({ providers: { external: ['default'], deepseek: ['v4-flash'] } })
const bot = { id: 'b1', model: 'deepseek/v4-flash', appliedModel: 'deepseek/v4-flash' }
rt.state.bots = { b1: bot }
await rt.catalog()
await rt.syncModels()
await rt.routeRefused('external/default')
// The Session drifted meanwhile — its own picker can move it without telling us.
delete bot.appliedModel
await rt.syncModels()
assert.deepEqual(selected, ['b1:deepseek/v4-flash'])
})
// A refused route is chosen again only once its provider serves other models: the same
// placeholder registered again, even after it was unregistered, is still not an endpoint.
test('an adapter update ends a refusal only when its provider serves other models', async () => {
const { rt, providers, listeners } = bench({ providers: { external: ['default'], deepseek: ['v4-flash'] }, fallback: { provider: 'external', model: 'default' } })
const adaptersUpdated = () => listeners.get('llm/adapters-updated')()
await rt.routeRefused('external/default')
await adaptersUpdated()
assert.equal(rt.defaultRef(), undefined)
delete providers.external
await adaptersUpdated()
providers.external = ['default']
await adaptersUpdated()
assert.equal(rt.defaultRef(), undefined)
providers.external = ['default', 'real']
await adaptersUpdated()
assert.equal(rt.defaultRef(), 'external/default')
assert.ok((await rt.catalog()).some(entry => entry.ref === 'external/default'))
})
test('a greeting waits for the provider of its Bot model to register', async () => {
const { rt, providers, listeners } = bench({ providers: {} })
const waiting = rt.waitForModel({ model: 'stepfun/step-5' }, 5000)
setTimeout(() => {
providers.stepfun = ['step-5']
listeners.get('llm/adapters-updated')?.()
}, 20)
assert.equal(await waiting, 'stepfun/step-5')
assert.equal(await rt.waitForModel({}, 5000), undefined)
})
test('a key set or a sign-in lifts what blocked a Bot; a key removed, a sign-out, a provider or a settings change only re-reads', async () => {
const keys = new Set(['DEEPSEEK_API_KEY'])
const records = new Set(['deepseek-account-platform/default'])
const credentials = {
describe: async ref => ({ configured: keys.has(ref), writable: true }),
describeRecord: async key => ({ configured: records.has(key), writable: true }),
}
const { rt, listeners, providers } = bench({ services: { credentials } })
const lifted = []
// Each listing also passes the providers listed with their key (the next test).
rt.unblockBots = fix => { if (fix.keyed === undefined) lifted.push(fix) }
await rt.catalog()
providers.deepseek = ['v4-pro']
await listeners.get('credentials/reference-updated')('DEEPSEEK_API_KEY')
assert.ok((await rt.catalog()).some(entry => entry.ref === 'deepseek/v4-pro'))
await listeners.get('credentials/reference-updated')('STEPFUN_API_KEY')
// Another plugin's record, such as an OAuth token refresh, is no sign-in.
await listeners.get('credentials/record-updated')('llm-pi-ai/github-copilot')
providers.deepseek = ['v4-flash']
await listeners.get('credentials/record-updated')('deepseek-account-platform/default')
assert.ok((await rt.catalog()).some(entry => entry.ref === 'deepseek/v4-flash'))
// The provider refused the token and its record went.
records.clear()
await listeners.get('credentials/record-updated')('deepseek-account-platform/default')
listeners.get('llm/adapters-updated')()
listeners.get('settings/document-updated')()
assert.deepEqual(lifted, [{ ref: 'DEEPSEEK_API_KEY' }, { signedIn: true }])
})
test('every listing passes the providers listed with their key, so a key that came without an event still lifts', async () => {
const { rt, providers } = bench()
providers.stepfun = ['step-5']
rt.missingKeys = async () => new Set(['stepfun'])
const passed = []
rt.unblockBots = fix => passed.push([...fix.keyed].sort())
await rt.catalog({ fresh: true })
rt.missingKeys = async () => new Set()
await rt.catalog({ fresh: true })
assert.equal(passed.length, 2)
assert.equal(passed[0].includes('stepfun'), false)
assert.equal(passed[1].includes('stepfun'), true)
})
test('a credential DSH cannot describe counts as there, so its fix still lifts', async () => {
const { rt, listeners } = bench({ services: { credentials: { describe: async () => { throw new Error('locked') } } } })
const lifted = []
rt.unblockBots = fix => { if (fix.keyed === undefined) lifted.push(fix) }
await listeners.get('credentials/reference-updated')('DEEPSEEK_API_KEY')
await listeners.get('credentials/record-updated')('deepseek-account-platform/default')
assert.deepEqual(lifted, [{ ref: 'DEEPSEEK_API_KEY' }, { signedIn: true }])
})
test('a Bot on the default model with its own thinking level leaves the default level as it was', async () => {
const fallback = { provider: 'deepseek', model: 'v4-flash', reasoningEffort: 'high' }
const { rt, current } = bench({ fallback, reasoning: { 'deepseek/v4-flash': { reasoning: { efforts: [{ id: 'low', name: 'Low' }, { id: 'high', name: 'High' }], defaultEffort: 'high' } } } })
rt.ctx.sessionController.selectModel = async ({ provider, model, reasoningEffort }) => {
const selected = { provider, model, ...(reasoningEffort === undefined ? {} : { reasoningEffort }) }
await rt.ctx.get('agentDefaultModel').saveSelection(selected)
return { selected }
}
await rt.applyModel({ id: 'b1', model: 'deepseek/v4-flash', effort: 'low' })
assert.deepEqual(current(), fallback)
})