A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Keep mailbox connections recoverable and deduplicate account adds
Fei24113166031549bd
diff --git a/ds-bot/src/host/connectors/mail/index.js b/ds-bot/src/host/connectors/mail/index.jsindex aa9b446d..62ff41fd 100644--- a/ds-bot/src/host/connectors/mail/index.js+++ b/ds-bot/src/host/connectors/mail/index.js@@ -17,6 +17,8 @@ const MAX_REQUESTS = 20export function install(rt, host) {const { state } = rtconst sessions = new Map()+ const connecting = new Map()+ const adding = new Map()const accounts = () => state.connectors?.mail?.accounts ?? {}const accountList = () => Object.values(accounts())@@ -24,7 +26,7 @@ export function install(rt, host) {const accountOf = (ref) => {const list = accountList()if (ref === undefined || ref === '') return list.length === 1 ? list[0] : undefined- return list.find(account => account.address === String(ref).toLowerCase() || account.id === ref)+ return list.find(account => account.address.toLowerCase() === String(ref).toLowerCase() || account.id === ref)}const googleClient = () => ({@@ -33,10 +35,26 @@ export function install(rt, host) {})const googleReady = () => Boolean(googleClient().id && googleClient().secret)+ async function openImap(account, password) {+ const factory = rt.config.mail?.connect ?? connectImap+ let session+ session = await factory(account, { password, onGone: () => {+ if (sessions.get(account.id) === session) sessions.delete(account.id)+ } })+ if (session.usable === false) throw new Error('The mail connection closed during verification')+ sessions.set(account.id, session)+ return session+ }+async function connect(account) {const key = account.idconst live = sessions.get(key)- if (live !== undefined) return live+ if (live !== undefined && live.usable !== false) return live+ if (live !== undefined) {+ sessions.delete(key)+ await live.close?.()+ }+ if (connecting.has(key)) return connecting.get(key)const make = async () => {if (account.auth === 'google') {const token = () => rt.secretValue(mailSecretName(account.address, 'TOKEN'))@@ -46,17 +64,13 @@ export function install(rt, host) {}const password = rt.secretValue(mailSecretName(account.address, 'PASSWORD'))if (password === undefined) throw new Error(`The password for ${account.address} is not saved`)- const factory = rt.config.mail?.connect ?? connectImap- const session = await factory(account, { password })- sessions.set(key, session)- return session+ return openImap(account, password)}+ const pending = make()+ connecting.set(key, pending)try {- return await make()- } catch (error) {- sessions.delete(key)- throw error- }+ return await pending+ } finally { connecting.delete(key) }}// Gmail tokens refresh through the same OAuth rules as connectors.@@ -281,30 +295,43 @@ export function install(rt, host) {if (!googleReady()) throw new Error('Google sign-in is not available in this build')return beginGoogle(address)}- const password = String(body.password ?? '').trim()- if (password === '') throw new Error('Enter the authorization code or app password')- const preset = presetOf(address)- const provider = typeof body.provider === 'string' ? body.provider : preset?.id ?? 'other'- const chosen = PRESETS.find(p => p.id === provider)- const imap = chosen && provider !== 'other' ? chosen.imap : { host: String(body.imapHost ?? '').trim(), port: Number(body.imapPort) || 993, secure: body.imapTls !== false }- const smtp = chosen && provider !== 'other' ? chosen.smtp : { host: String(body.smtpHost ?? '').trim(), port: Number(body.smtpPort) || 465, secure: body.smtpTls !== false }- if (imap.host === '' || smtp.host === '') throw new Error('Enter the IMAP and SMTP hosts')- const id = `m${randomUUID().slice(0, 8)}`- const account = { id, address, provider, auth: 'password', imap, smtp, createdAt: Date.now() }- // Log in once before saving, so a bad password never lands in state.- const factory = rt.config.mail?.connect ?? connectImap- let session- try {- session = await factory(account, { password })- } catch (error) {- throw new Error(mailError(error))- }- state.connectors = { ...(state.connectors ?? {}), mail: { ...state.connectors?.mail, accounts: { ...accounts(), [id]: account } } }- await host.secrets.writeSecret(mailSecretName(address, 'PASSWORD'), password, `Mailbox ${address}`)- await rt.save()- sessions.set(id, session)- syncTools()- return mailView()+ if (adding.has(address)) return adding.get(address)+ const pending = (async () => {+ const password = String(body.password ?? '').trim()+ if (password === '') throw new Error('Enter the authorization code or app password')+ const preset = presetOf(address)+ const provider = typeof body.provider === 'string' ? body.provider : preset?.id ?? 'other'+ const chosen = PRESETS.find(p => p.id === provider)+ const imap = chosen && provider !== 'other' ? chosen.imap : { host: String(body.imapHost ?? '').trim(), port: Number(body.imapPort) || 993, secure: body.imapTls !== false }+ const smtp = chosen && provider !== 'other' ? chosen.smtp : { host: String(body.smtpHost ?? '').trim(), port: Number(body.smtpPort) || 465, secure: body.smtpTls !== false }+ if (imap.host === '' || smtp.host === '') throw new Error('Enter the IMAP and SMTP hosts')+ const existing = accountOf(address)+ const id = existing?.id ?? `m${randomUUID().slice(0, 8)}`+ const account = { ...existing, id, address, provider, auth: 'password', imap, smtp, createdAt: existing?.createdAt ?? Date.now() }+ delete account.error+ // Log in once before saving, so a bad password never lands in state.+ const old = sessions.get(id)+ let session+ try {+ session = await openImap(account, password)+ } catch (error) {+ throw new Error(mailError(error))+ }+ await old?.close?.().catch(() => {})+ try {+ await host.secrets.writeSecret(mailSecretName(address, 'PASSWORD'), password, `Mailbox ${address}`)+ state.connectors = { ...(state.connectors ?? {}), mail: { ...state.connectors?.mail, accounts: { ...accounts(), [id]: account } } }+ await rt.save()+ } catch (error) {+ if (sessions.get(id) === session) sessions.delete(id)+ await session.close?.()+ throw error+ }+ syncTools()+ return mailView()+ })()+ adding.set(address, pending)+ try { return await pending } finally { adding.delete(address) }},async 'mail-account-remove'(body) {const account = accountOf(body.id ?? body.address)diff --git a/ds-bot/src/host/connectors/mail/transport.js b/ds-bot/src/host/connectors/mail/transport.jsindex 74a65e2e..1e374621 100644--- a/ds-bot/src/host/connectors/mail/transport.js+++ b/ds-bot/src/host/connectors/mail/transport.js@@ -2,6 +2,7 @@// behind a small interface the tools and tests share; tests inject fakes// through rt.config.mail.connect.import { simpleParser } from 'mailparser'+import { Socket } from 'node:net'import { IMAP_ID } from './presets.js'// IMAP + SMTP (imapflow + nodemailer). One live IMAP connection per account is@@ -17,16 +18,61 @@ export const imapOptions = (account, deps = {}) => ({})export async function connectImap(account, deps = {}) {- const { ImapFlow } = await import('imapflow')- const nodemailer = await import('nodemailer')+ const { ImapFlow } = deps.modules ?? await import('imapflow')+ const nodemailer = deps.modules?.nodemailer ?? await import('nodemailer')const imap = new ImapFlow(imapOptions(account, deps))- const smtp = nodemailer.createTransport({+ let gone = false+ let smtp+ const sockets = new Set()+ const retire = () => {+ if (gone) return+ gone = true+ imap.close()+ for (const socket of sockets) socket.destroy()+ smtp?.close()+ deps.onGone?.()+ }+ imap.on('error', retire)+ imap.on('close', retire)+ smtp = nodemailer.createTransport({host: account.smtp.host, port: account.smtp.port, secure: account.smtp.secure === true,requireTLS: account.smtp.starttls === true,auth: { user: account.address, pass: deps.password },+ // Nodemailer's close() does not cancel a non-pooled verify(). Own its sockets+ // through the transport hook so an IMAP failure also cancels that login.+ getSocket(options, done) {+ if (gone) { done(new Error('The mail connection is closed')); return }+ const socket = new Socket()+ sockets.add(socket)+ let answered = false+ const fail = error => { if (!answered) { answered = true; done(error) } }+ socket.once('error', fail)+ socket.on('error', retire)+ socket.once('close', () => {+ sockets.delete(socket)+ fail(new Error('The SMTP connection closed'))+ })+ // Preserve Nodemailer's documented default connection timeout.+ socket.setTimeout(options.connectionTimeout ?? 120_000, () => socket.destroy(new Error('SMTP connection timeout')))+ socket.connect({ host: options.host, port: options.port }, () => {+ if (answered) return+ answered = true+ socket.setTimeout(0)+ done(null, { connection: socket })+ })+ },})- await imap.connect()- await smtp.verify()+ smtp.on('error', retire)+ try {+ await Promise.all([+ imap.connect().then(() => { if (gone) imap.close() }),+ smtp.verify().then(() => { if (gone) smtp.close() }),+ ])+ if (gone) throw new Error('The mail connection closed during verification')+ } catch (error) {+ retire()+ throw error+ }const folders = async () => {const list = await imap.list()@@ -75,7 +121,8 @@ export async function connectImap(account, deps = {}) {return {kind: 'imap',- async close() { try { await imap.logout() } catch { /* dropped */ } },+ get usable() { return !gone && imap.usable },+ async close() { try { if (!gone) await imap.logout() } catch { /* dropped */ } finally { retire() } },async search({ folder = 'INBOX', from, to, subject, text, since, before, unread, limit = 20 }) {const query = {}if (from) query.from = fromdiff --git a/ds-bot/test/connectors-engine.test.js b/ds-bot/test/connectors-engine.test.jsindex 6ec20281..01b1af93 100644--- a/ds-bot/test/connectors-engine.test.js+++ b/ds-bot/test/connectors-engine.test.js@@ -3,6 +3,7 @@// test process (fakes listen on 127.0.0.1 or answer in memory).import assert from 'node:assert/strict'import { createServer } from 'node:http'+import { EventEmitter } from 'node:events'import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'import { tmpdir } from 'node:os'import { join } from 'node:path'@@ -16,6 +17,7 @@ const oauth = await import('../src/host/connectors/oauth.js')const { createGateway } = await import('../src/host/connectors/gateway.js')const { validateCapabilityName } = await import('../src/host/capabilities.js')const { mailSecretName } = await import('../src/host/connectors/mail/presets.js')+const { connectImap } = await import('../src/host/connectors/mail/transport.js')const addressAt = domain => `${'me'}@${domain}`@@ -461,6 +463,117 @@ const fakeSession = () => {}}+test('mail_search reconnects once after an idle drop or an unusable cached session', async (t) => {+ const made = []+ const factory = async (_account, { onGone }) => {+ const session = { ...fakeSession(), usable: true, gone: () => onGone() }+ made.push(session)+ return session+ }+ const home = teamHome()+ const ctx = fakeContext()+ const rt = createRuntime(ctx, { home, github: false, mail: { connect: factory } })+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true }) })+ await rt.load()+ await rt.connectorEndpoint('mail-account-add', { address: addressAt('qq.com'), password: 'CODE' })+ const search = rt.teamTools.find(d => d.name === 'mail_search')+ made[0].gone()+ assert.equal(await search.execute({}, { agent: { id: 'm1' } }), 'No matching mail.')+ assert.equal(made.length, 2)+ made[0].gone() // A late close from the old connection must not drop the new one.+ await search.execute({}, { agent: { id: 'm1' } })+ assert.equal(made.length, 2)+ made[1].usable = false+ await search.execute({}, { agent: { id: 'm1' } })+ assert.equal(made.length, 3)+})++test('an idle ImapFlow error cannot crash the engine and the next search logs in once', async (t) => {+ const clients = []+ class ImapFlow extends EventEmitter {+ usable = false+ constructor() { super(); clients.push(this) }+ async connect() { this.usable = true }+ close() { this.usable = false; this.emit('close') }+ async logout() { this.close() }+ async getMailboxLock() { return { release() {} } }+ async search() { return [] }+ }+ let made = 0+ const factory = (account, deps) => {+ made += 1+ return connectImap(account, { ...deps, modules: {+ ImapFlow, nodemailer: { createTransport: () => Object.assign(new EventEmitter(), { verify: async () => {}, close() {} }) },+ } })+ }+ const home = teamHome()+ const ctx = fakeContext()+ const rt = createRuntime(ctx, { home, github: false, mail: { connect: factory } })+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true }) })+ await rt.load()+ await rt.connectorEndpoint('mail-account-add', { address: addressAt('qq.com'), password: 'CODE' })+ assert.doesNotThrow(() => clients[0].emit('error', new Error('Socket timeout')))+ assert.equal(await rt.teamTools.find(d => d.name === 'mail_search').execute({}, { agent: { id: 'm1' } }), 'No matching mail.')+ assert.equal(made, 2)+})++test('mail add updates the same address and keeps its verified session for tools', async (t) => {+ const made = []+ const home = teamHome()+ const ctx = fakeContext()+ const rt = createRuntime(ctx, { home, github: false, mail: { connect: async (account, deps) => {+ const session = fakeSession()+ made.push({ account, password: deps.password, session })+ return session+ } } })+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true }) })+ await rt.load()+ const first = await rt.connectorEndpoint('mail-account-add', { address: addressAt('qq.com'), password: 'FIRST' })+ const id = first.accounts[0].id+ const search = rt.teamTools.find(d => d.name === 'mail_search')+ await search.execute({}, { agent: { id: 'm1' } })+ assert.equal(made.length, 1)+ assert.equal(made[0].session.calls.some(c => c[0] === 'close'), false)+ const updated = await rt.connectorEndpoint('mail-account-add', {+ address: addressAt('qq.com').toUpperCase(), password: 'SECOND', provider: 'other',+ imapHost: 'imap.example.test', smtpHost: 'smtp.example.test',+ })+ assert.equal(updated.accounts.length, 1)+ assert.equal(updated.accounts[0].id, id)+ assert.equal(made.length, 2)+ assert.equal(made[1].account.imap.host, 'imap.example.test')+ assert.equal(rt.secretValue(mailSecretName(addressAt('qq.com'), 'PASSWORD')), 'SECOND')+ assert.equal(made[0].session.calls.some(c => c[0] === 'close'), true)+ await search.execute({}, { agent: { id: 'm1' } })+ assert.equal(made.length, 2)+ assert.equal(made[1].session.calls.some(c => c[0] === 'search'), true)+})++test('concurrent adds for one address share verification and its result', async (t) => {+ const gate = Promise.withResolvers()+ const started = Promise.withResolvers()+ let made = 0+ const home = teamHome()+ const ctx = fakeContext()+ const rt = createRuntime(ctx, { home, github: false, mail: { connect: async () => {+ made += 1+ started.resolve()+ await gate.promise+ return fakeSession()+ } } })+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true }) })+ await rt.load()+ const first = rt.connectorEndpoint('mail-account-add', { address: addressAt('qq.com'), password: 'CODE' })+ await started.promise+ const second = rt.connectorEndpoint('mail-account-add', { address: addressAt('qq.com').toUpperCase(), password: 'CODE' })+ await new Promise(resolve => setImmediate(resolve))+ gate.resolve()+ const [a, b] = await Promise.all([first, second])+ assert.equal(made, 1)+ assert.deepEqual(a, b)+ assert.equal(rt.mailView().accounts.length, 1)+})+test('mail account add verifies before saving; a bad code fails cleanly; send needs the card', async (t) => {const sessions = new Map()const factory = async (account, deps) => {diff --git a/ds-bot/test/connectors-mail-transport.test.js b/ds-bot/test/connectors-mail-transport.test.jsnew file mode 100644index 00000000..a1b916c7--- /dev/null+++ b/ds-bot/test/connectors-mail-transport.test.js@@ -0,0 +1,135 @@+import assert from 'node:assert/strict'+import { EventEmitter } from 'node:events'+import { createServer } from 'node:net'+import { test } from 'node:test'+import { connectImap } from '../src/host/connectors/mail/transport.js'++const account = {+ address: '[email protected]',+ imap: { host: '127.0.0.1', port: 1 },+ smtp: { host: '127.0.0.1', port: 1 },+}++const transport = ({ connect = async () => {}, verify = async () => {} } = {}) => {+ let imap+ class ImapFlow extends EventEmitter {+ usable = false+ closed = false+ constructor() { super(); imap = this }+ async connect() { await connect(); this.usable = true }+ close() { this.closed = true; this.usable = false; this.emit('close') }+ async logout() { this.close() }+ }+ const smtp = new EventEmitter()+ smtp.verify = verify+ smtp.closed = false+ smtp.close = () => { smtp.closed = true }+ return { modules: { ImapFlow, nodemailer: { createTransport: () => smtp } }, get imap() { return imap }, smtp }+}++test('idle IMAP errors and close retire the session without escaping', async () => {+ const fake = transport()+ let gone = 0+ const session = await connectImap(account, { modules: fake.modules, onGone: () => { gone += 1 } })+ assert.equal(session.usable, true)+ assert.doesNotThrow(() => fake.imap.emit('error', new Error('Socket timeout')))+ assert.equal(session.usable, false)+ fake.imap.emit('close')+ assert.equal(gone, 1)+ assert.equal(fake.smtp.closed, true)+})++test('SMTP errors cannot escape and close both transports', async () => {+ const fake = transport()+ const session = await connectImap(account, { modules: fake.modules })+ assert.doesNotThrow(() => fake.smtp.emit('error', new Error('SMTP dropped')))+ assert.equal(session.usable, false)+ assert.equal(fake.imap.closed, true)+ assert.equal(fake.smtp.closed, true)+})++test('IMAP connect and SMTP verify start together', async () => {+ const imapGate = Promise.withResolvers()+ const smtpStarted = Promise.withResolvers()+ const fake = transport({ connect: () => imapGate.promise, verify: async () => { smtpStarted.resolve() } })+ const pending = connectImap(account, { modules: fake.modules })+ try {+ await Promise.race([smtpStarted.promise, new Promise((_, reject) => setTimeout(() => reject(new Error('SMTP waited for IMAP')), 100))])+ } finally { imapGate.resolve(); await (await pending).close() }+})++for (const side of ['IMAP', 'SMTP']) {+ test(`${side} verification failure closes both transports, including a late connection`, async () => {+ const gate = Promise.withResolvers()+ const fail = async () => { throw new Error(`${side} refused`) }+ const fake = transport({+ connect: side === 'IMAP' ? fail : () => gate.promise,+ verify: side === 'SMTP' ? fail : () => gate.promise,+ })+ const pending = connectImap(account, { modules: fake.modules })+ try {+ await assert.rejects(Promise.race([+ pending,+ new Promise((_, reject) => setTimeout(() => reject(new Error('verification did not reject')), 100)),+ ]), new RegExp(`${side} refused`))+ } finally { gate.resolve(); await pending.catch(() => {}) }+ await new Promise(resolve => setImmediate(resolve))+ assert.equal(fake.imap.closed, true)+ assert.equal(fake.imap.usable, false)+ assert.equal(fake.smtp.closed, true)+ })+}++test('an IMAP failure cancels a real nodemailer verification socket', async (t) => {+ const accepted = Promise.withResolvers()+ const closed = Promise.withResolvers()+ const server = createServer(socket => {+ socket.on('error', () => {})+ socket.on('close', () => closed.resolve())+ accepted.resolve()+ // Deliberately never send an SMTP greeting.+ })+ await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))+ t.after(() => server.close())+ const fake = transport({ connect: async () => { await accepted.promise; throw new Error('IMAP refused') } })+ const nodemailer = await import('nodemailer')+ await assert.rejects(connectImap({+ ...account, smtp: { host: '127.0.0.1', port: server.address().port, secure: false },+ }, { modules: { ImapFlow: fake.modules.ImapFlow, nodemailer } }), /IMAP refused/)+ await closed.promise+})++test('a verified SMTP transport opens a fresh socket when sending', async (t) => {+ let connections = 0+ const server = createServer(socket => {+ connections += 1+ socket.on('error', () => {})+ socket.write('220 fake SMTP\r\n')+ let data = ''+ let body = false+ socket.on('data', chunk => {+ data += chunk.toString()+ let end+ while ((end = data.indexOf('\r\n')) !== -1) {+ const line = data.slice(0, end)+ data = data.slice(end + 2)+ if (body) {+ if (line === '.') { body = false; socket.write('250 queued\r\n') }+ } else if (line.startsWith('DATA')) { body = true; socket.write('354 send data\r\n') }+ else if (line.startsWith('QUIT')) socket.end('221 bye\r\n')+ else socket.write('250 OK\r\n')+ }+ })+ })+ await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))+ t.after(() => server.close())+ const fake = transport()+ const nodemailer = await import('nodemailer')+ const session = await connectImap({+ ...account, smtp: { host: '127.0.0.1', port: server.address().port, secure: false },+ }, { modules: { ImapFlow: fake.modules.ImapFlow, nodemailer } })+ t.after(() => session.close())+ await session.send({ to: '[email protected]', subject: 'Test', text: 'body' })+ assert.equal(connections, 2)+ assert.equal(session.usable, true)+})