A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
// The first-run import: a copy of the Bot team in the user's own DSH home. It only reads there.
import { homedir, hostname } from 'node:os'
import { join, relative, resolve } from 'node:path'
import { parseSessionFormatLogFilename } from '@deepseek-ai/dsh-session-format'
import { sessionFormatCatalog } from '@deepseek-ai/dsh-session-format-catalog'
import { parse, stringify } from 'yaml'
import { copyTree, disk, within } from './copy-tree.mjs'
export { within }
const { existsSync, readdirSync, readFileSync, statSync } = disk
const { chmod, cp, lstat, mkdir, readdir, readFile, rename, rm, writeFile } = disk.promises
// The rows of a profile patch that hold model settings: DeepSeek's API and account, other
// providers, the default model and the agent presets. Rows of other plugins may name plugins
// that the app does not have.
export const MODEL_ROWS = ['llm-deepseek', 'llm-deepseek-account', 'llm-pi-ai', 'agent-default-model', 'agent-preset-registry']
// The team's lock, its turns in flight and its workspace belong to the old home.
const LEFT_BEHIND = new Set(['state.lock', 'inflight.json', 'workspace'])
const LOCK_STALE_MS = 60_000
// Folders that a project makes again from its own files (windows-onboarding §4): the import card
// offers to leave them out of the work files, and does so by default from 200 MB on.
export const REGENERABLE = new Set(['node_modules', '.venv', '__pycache__', '.pnpm-store', '.next'])
export const REGENERABLE_DEFAULT_BYTES = 200e6
/** The DSH home of the user's own DSH, read as DSH reads DSH_HOME. */
export function userDshHome(env = process.env) {
const value = env.DSH_HOME?.trim()
if (!value) return join(homedir(), '.dsh')
return resolve(value === '~' ? homedir() : /^~[\\/]/u.test(value) ? join(homedir(), value.slice(2)) : value)
}
const readJson = path => JSON.parse(readFileSync(path, 'utf8'))
/** Every Session the team names: each Bot's and group's parts, the group conversations and each group's own Session. */
export function sessionIdsOf(team) {
const ids = segments => segments.map(part => part?.sessionId)
return [...new Set([
...Object.values(team.bots ?? {}).flatMap(bot => [bot.id, bot.sessionId, ...ids(bot.segments ?? [])]),
// A group keeps the earlier parts per member: { [botId]: segments }. The group's own
// Session, the one it relays in, has the group's id.
...Object.values(team.rooms ?? {}).flatMap(room => [room.id, ...Object.values(room.sessions ?? {}), ...ids(Object.values(room.segments ?? {}).flat())]),
].filter(id => typeof id === 'string' && id !== ''))]
}
/**
* The team in a DSH home, with the profiles that hold model settings.
* @returns {{ from: string, team: any, bots: number, rooms: number, profiles: { name: string, rows: any[] }[] } | undefined}
*/
export function findTeam(from) {
const statePath = join(from, 'bot', 'state.json')
if (!existsSync(statePath)) return undefined
const team = readJson(statePath)
const bots = Object.keys(team.bots ?? {}).length
if (bots === 0) return undefined
return { from, team, bots, rooms: Object.keys(team.rooms ?? {}).length, profiles: modelProfiles(from) }
}
function modelProfiles(from) {
const dir = join(from, 'profiles')
if (!existsSync(dir)) return []
return readdirSync(dir, { withFileTypes: true }).filter(entry => entry.isDirectory()).flatMap((entry) => {
const file = join(dir, entry.name, 'cordis.patch.yml')
if (!existsSync(file)) return []
let patch
try {
patch = parse(readFileSync(file, 'utf8'))
} catch (error) {
// An unreadable profile has no model settings to copy; the team is still there.
console.warn(`DS Bot: skipped the model settings in ${file}: ${error instanceof Error ? error.message : String(error)}`)
return []
}
const rows = Array.isArray(patch) ? patch.filter(row => MODEL_ROWS.includes(row?.id)) : []
return rows.length > 0 ? [{ name: entry.name, rows }] : []
})
}
/**
* Whether a running DSH holds the team, by the rule of ds-bot's own lock (src/host/home-lock.js):
* the holder bumps the lock's mtime, so a stale lock is a holder that is gone, even when its pid
* now belongs to another process.
*/
export function teamInUse(from, { host = hostname(), now = Date.now(), isAlive = pidAlive } = {}) {
const path = join(from, 'bot', 'state.lock')
const lock = statSync(path, { throwIfNoEntry: false })
if (lock === undefined || now - lock.mtimeMs >= LOCK_STALE_MS) return false
let holder
try {
holder = JSON.parse(readFileSync(path, 'utf8'))
} catch {
// Being written right now.
return true
}
return holder?.host !== host || isAlive(holder.pid)
}
function pidAlive(pid) {
try {
process.kill(pid, 0)
return true
} catch (error) {
// EPERM: the process exists but belongs to someone else.
return error?.code === 'EPERM'
}
}
// sessions/<group>/<id>/: the group is the folder the Session works in.
function sessionDirs(from, ids) {
const root = join(from, 'sessions')
const groups = existsSync(root) ? readdirSync(root, { withFileTypes: true }).filter(entry => entry.isDirectory()).map(entry => entry.name) : []
const found = []
const missing = []
for (const id of ids) {
const group = groups.find(name => existsSync(join(root, name, id)))
if (group === undefined) missing.push(id)
else found.push({ id, group })
}
return { found, missing }
}
// fs.cp that tells `seen` the size of each file it is about to copy.
function cpSeen(source, target, { filter = () => true, seen, ...options }) {
return cp(source, target, {
...options,
filter: async (path) => {
if (!await filter(path)) return false
if (seen !== undefined) {
const info = await lstat(path).catch(() => undefined)
if (info?.isFile()) seen(info.size)
}
return true
},
})
}
async function copySessionDirs(from, to, sessions, signal, seen, each = () => {}) {
for (const [index, { id, group }] of sessions.entries()) {
signal?.throwIfAborted()
await cpSeen(join(from, 'sessions', group, id), join(to, 'sessions', group, id), { recursive: true, seen })
each(index + 1, sessions.length)
}
}
async function moveSessionDirs(stage, to, sessions, signal) {
for (const { id, group } of sessions) {
signal?.throwIfAborted()
await mkdir(join(to, 'sessions', group), { recursive: true })
// A copy that stopped can have moved the Session in already.
await rm(join(to, 'sessions', group, id), { recursive: true, force: true })
await rename(join(stage, 'sessions', group, id), join(to, 'sessions', group, id))
}
}
/**
* Copy the team's Sessions into the DSH home `to`, as they lie in `from`; an abort stops it between Sessions.
* @param {string} from @param {any} team @param {string} to @param {AbortSignal} [signal]
* @param {(done: number, total: number) => void} [onProgress] after each Session
* @returns {Promise<string[]>} their ids
*/
export async function copySessions(from, team, to, signal, onProgress) {
const { found } = sessionDirs(from, sessionIdsOf(team))
await copySessionDirs(from, to, found, signal, undefined, onProgress)
return found.map(({ id }) => id)
}
// Thrown when measuring runs out of time.
const OVER = Symbol('over budget')
// The bytes under `path` into `tally`, links counted as themselves and what cannot be read as
// nothing; `tally.regen` takes the bytes inside the folders that REGENERABLE names.
async function measure(path, clock, tally, regen = false) {
if (clock.now() > clock.until) throw OVER
const entry = await lstat(path).catch(() => undefined)
if (entry === undefined) return
if (!entry.isDirectory()) {
tally.bytes += entry.size
if (regen) tally.regen += entry.size
return
}
for (const name of await readdir(path).catch(() => [])) await measure(join(path, name), clock, tally, regen || REGENERABLE.has(name))
}
/**
* What the import card says about the team (windows-onboarding §4): how many of its Sessions are
* there, the bytes of its chats (the team's own files but its workspace, its Sessions and the
* attachments), of its work files (the team workspace) and of the regenerable folders among
* them. The sizes are left out when measuring takes longer than `budget` ms: a team workspace can
* hold many a node_modules.
* @returns {Promise<{ sessions: number, chats?: number, work?: number, regen?: number }>}
*/
export async function teamSizes(from, team, { budget = 2000, now = Date.now } = {}) {
const { found } = sessionDirs(from, sessionIdsOf(team))
const clock = { until: now() + budget, now }
const chats = { bytes: 0, regen: 0 }
const work = { bytes: 0, regen: 0 }
try {
const bot = join(from, 'bot')
for (const name of await readdir(bot).catch(() => [])) {
if (!LEFT_BEHIND.has(name)) await measure(join(bot, name), clock, chats)
}
for (const { id, group } of found) await measure(join(from, 'sessions', group, id), clock, chats)
await measure(join(from, 'attachments'), clock, chats)
await measure(join(bot, 'workspace'), clock, work)
} catch (error) {
if (error === OVER) return { sessions: found.length }
throw error
}
return { sessions: found.length, chats: chats.bytes, work: work.bytes, regen: work.regen }
}
const logVersion = name => parseSessionFormatLogFilename(name.replace(/\.(zstd|zst|gz|br)$/u, ''))
const isNewer = (from, { id, group }) => readdirSync(join(from, 'sessions', group, id))
.some(name => (logVersion(name) ?? 0) > sessionFormatCatalog.currentVersion)
/** The Sessions whose log is in a newer format than the pinned DSH reads. */
export function newerSessions(from, team) {
const { found } = sessionDirs(from, sessionIdsOf(team))
return found.filter(session => isNewer(from, session)).map(({ id }) => id)
}
/**
* Copy into the data location `to` the Sessions its team names that are not there, from the DSH
* home the team was imported from. Imports before 0.2.3 left out each group's own Session, and
* such a group does not open (#551). Each Session is put together beside the data and moved in
* whole; one this DSH cannot read stays where it is. Nothing in the old home changes.
* @param {string} to @param {AbortSignal} [signal]
* @returns {Promise<{ copied: string[], newer: string[] }>} the ids copied, and those left for their newer format
*/
export async function restoreSessions(to, signal) {
const record = join(to, 'bot', 'imported.json')
const statePath = join(to, 'bot', 'state.json')
const none = { copied: [], newer: [] }
if (!existsSync(record) || !existsSync(statePath)) return none
const { from } = readJson(record)
if (typeof from !== 'string' || resolve(from) === resolve(to)) return none
for (const name of readdirSync(to).filter(entry => /^\.restore-\d+$/u.test(entry))) {
await rm(join(to, name), { recursive: true, force: true })
}
const { missing } = sessionDirs(to, sessionIdsOf(readJson(statePath)))
const { found } = sessionDirs(from, missing)
const newer = found.filter(session => isNewer(from, session))
const sessions = found.filter(session => !newer.includes(session))
if (sessions.length > 0) {
const stage = join(to, `.restore-${String(Date.now())}`)
await copySessionDirs(from, stage, sessions, signal)
await moveSessionDirs(stage, to, sessions, signal)
await rm(stage, { recursive: true, force: true })
}
return { copied: sessions.map(({ id }) => id), newer: newer.map(({ id }) => id) }
}
// The Host's first start on the data location writes a .credentials.yaml of its own: the browser
// session grant of the connection plugin, under client-connection/. A file with nothing else in
// it holds no key of the user's.
async function onlyHostRecords(file) {
let document
try {
document = parse(await readFile(file, 'utf8')) ?? {}
} catch {
// A file that does not read as DSH's is the user's to fix, not ours to replace.
return false
}
if (typeof document !== 'object' || Array.isArray(document)) return false
const { version, refs, records, ...other } = document
return (version === undefined || version === 1) && Object.keys(other).length === 0
&& Object.keys(refs ?? {}).length === 0
&& Object.keys(records ?? {}).every(key => key.startsWith('client-connection/'))
}
// A key file the data location has already is the user's own, saved there since, and stays;
// a .credentials.yaml that only the Host wrote is not.
// Returns 'copied', 'kept', or undefined when the old home has none.
async function copyKey(from, to, name) {
const source = join(from, name)
if (!existsSync(source)) return undefined
const target = join(to, name)
const replace = name === '.credentials.yaml' && existsSync(target) && await onlyHostRecords(target)
try {
await cp(source, target, { force: replace, errorOnExist: !replace })
} catch (error) {
if (error?.code === 'ERR_FS_CP_EEXIST') return 'kept'
throw error
}
if (process.platform !== 'win32') await chmod(target, 0o600)
return 'copied'
}
async function writeModelRows(to, rows) {
const file = join(to, 'profiles', 'ds-bot', 'cordis.patch.yml')
const current = existsSync(file) ? parse(await readFile(file, 'utf8')) : undefined
const ids = new Set(rows.map(row => row.id))
const kept = Array.isArray(current) ? current.filter(row => !ids.has(row?.id)) : []
await mkdir(join(to, 'profiles', 'ds-bot'), { recursive: true })
await writeFile(file, stringify([...kept, ...rows]))
}
/**
* Copy the team of `from` into the data location `to`, which the Host has started on once,
* so that DSH does not take the copied Sessions' folders for its own projects. The old team's
* workspace files go into `workspace`, without replacing a file that is there, and the key
* files only where the data location has none of the user's own. `leaveRegenerable` leaves the
* REGENERABLE folders out of the workspace files, `skipWork` all of them; `onProgress` hears the
* bytes copied so far as each file is about to be copied.
* Everything but the keys, the model settings and the workspace files is put together in a
* folder beside the data and moved in at the end. A copy that stops or fails removes that
* folder; one that was killed leaves it behind, and the next copy removes it.
* @param {{ from: string, to: string, workspace: string, profile?: string, dsh?: string, signal?: AbortSignal,
* leaveRegenerable?: boolean, skipWork?: boolean, onProgress?: (bytes: number) => void }} options
*/
export async function importTeam({ from, to, workspace, profile, dsh, signal, leaveRegenerable = false, skipWork = false, onProgress }) {
const found = findTeam(from)
if (found === undefined) throw new Error(`There is no Bot team in ${from}`)
const present = existsSync(join(to, 'bot', 'state.json')) ? readJson(join(to, 'bot', 'state.json')) : undefined
if (Object.keys(present?.bots ?? {}).length > 0) throw new Error(`${to} already has a Bot team`)
for (const name of (existsSync(to) ? readdirSync(to) : []).filter(entry => /^\.import-\d+$/u.test(entry))) {
await rm(join(to, name), { recursive: true, force: true })
}
const stage = join(to, `.import-${String(Date.now())}`)
try {
let bytes = 0
const seen = (size) => { bytes += size; onProgress?.(bytes) }
return await copyThroughStage({ found, from, to, workspace, profile, dsh, signal, stage, leaveRegenerable, skipWork, seen })
} finally {
await rm(stage, { recursive: true, force: true, maxRetries: 5 }).catch((error) => { console.error(error) })
}
}
async function copyThroughStage({ found, from, to, workspace, profile, dsh, signal, stage, leaveRegenerable, skipWork, seen }) {
const oldBot = join(from, 'bot')
signal?.throwIfAborted()
await cpSeen(oldBot, join(stage, 'bot'), { recursive: true, seen, filter: path => !LEFT_BEHIND.has(relative(oldBot, path).split(/[\\/]/u)[0]) })
const team = readJson(join(stage, 'bot', 'state.json'))
// The new data location registers its own team workspace. A DSH Agent's Sessions do not move
// to a fresh part, so they would go on working in the old team folder in the user's DSH home.
delete team.workspaceId
team.agents = {}
await writeFile(join(stage, 'bot', 'state.json'), JSON.stringify(team, null, 2))
const { found: sessions, missing } = sessionDirs(from, sessionIdsOf(found.team))
signal?.throwIfAborted()
await copySessionDirs(from, stage, sessions, signal, seen)
signal?.throwIfAborted()
if (existsSync(join(from, 'attachments'))) await cpSeen(join(from, 'attachments'), join(stage, 'attachments'), { recursive: true, seen })
signal?.throwIfAborted()
await mkdir(workspace, { recursive: true })
if (!skipWork && existsSync(join(oldBot, 'workspace'))) {
const leave = leaveRegenerable ? path => path.split(/[\\/]/u).some(name => REGENERABLE.has(name)) : undefined
// Bots that ran npm or pnpm there leave links, some of them to nothing.
const skipped = await copyTree(join(oldBot, 'workspace'), workspace, { force: false, skip: true, signal, leave, seen })
if (skipped.length > 0) console.warn('DS Bot: the import skipped what it could not read in the old team workspace:', skipped)
}
signal?.throwIfAborted()
const keys = []
const keptKeys = []
for (const name of ['.credentials.yaml', '.env']) {
const done = await copyKey(from, to, name)
if (done === 'copied') keys.push(name)
if (done === 'kept') keptKeys.push(name)
}
if (keptKeys.length > 0) console.warn(`DS Bot: the import kept the ${keptKeys.join(' and ')} that the data location has`)
const rows = found.profiles.find(entry => entry.name === profile)?.rows ?? []
if (rows.length > 0) await writeModelRows(to, rows)
await moveSessionDirs(stage, to, sessions, signal)
signal?.throwIfAborted()
if (existsSync(join(stage, 'attachments'))) await cp(join(stage, 'attachments'), join(to, 'attachments'), { recursive: true, force: false })
const summary = {
from, at: new Date().toISOString(), ...(dsh ? { dsh } : {}),
bots: found.bots, rooms: found.rooms, sessions: sessions.length, missingSessions: missing,
keys, keptKeys, modelSettings: rows.length > 0 ? profile : null,
...(skipWork ? { workFiles: 'skipped' } : leaveRegenerable ? { workFiles: 'regenerable left out' } : {}),
}
signal?.throwIfAborted()
await writeFile(join(stage, 'bot', 'imported.json'), `${JSON.stringify(summary, null, 2)}\n`)
// Until this rename the data has no Bot team, so a copy that stopped can run again.
await rm(join(to, 'bot'), { recursive: true, force: true })
await rename(join(stage, 'bot'), join(to, 'bot'))
return summary
}