Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

Keep mailbox connections recoverable and deduplicate account adds

Fei24113166031549bd
diff --git a/ds-bot/src/host/connectors/mail/index.js b/ds-bot/src/host/connectors/mail/index.js
index aa9b446d..62ff41fd 100644
--- a/ds-bot/src/host/connectors/mail/index.js
+++ b/ds-bot/src/host/connectors/mail/index.js
@@ -17,6 +17,8 @@ const MAX_REQUESTS = 20
export function install(rt, host) {
const { state } = rt
const sessions = new Map()
+ const connecting = new Map()
+ const adding = new Map()
const accounts = () => state.connectors?.mail?.accounts ?? {}
const accountList = () => Object.values(accounts())
@@ -24,7 +26,7 @@ export function install(rt, host) {
const accountOf = (ref) => {
const list = accountList()
if (ref === undefined || ref === '') return list.length === 1 ? list[0] : undefined
- return list.find(account => account.address === String(ref).toLowerCase() || account.id === ref)
+ return list.find(account => account.address.toLowerCase() === String(ref).toLowerCase() || account.id === ref)
}
const googleClient = () => ({
@@ -33,10 +35,26 @@ export function install(rt, host) {
})
const googleReady = () => Boolean(googleClient().id && googleClient().secret)
+ async function openImap(account, password) {
+ const factory = rt.config.mail?.connect ?? connectImap
+ let session
+ session = await factory(account, { password, onGone: () => {
+ if (sessions.get(account.id) === session) sessions.delete(account.id)
+ } })
+ if (session.usable === false) throw new Error('The mail connection closed during verification')
+ sessions.set(account.id, session)
+ return session
+ }
+
async function connect(account) {
const key = account.id
const live = sessions.get(key)
- if (live !== undefined) return live
+ if (live !== undefined && live.usable !== false) return live
+ if (live !== undefined) {
+ sessions.delete(key)
+ await live.close?.()
+ }
+ if (connecting.has(key)) return connecting.get(key)
const make = async () => {
if (account.auth === 'google') {
const token = () => rt.secretValue(mailSecretName(account.address, 'TOKEN'))
@@ -46,17 +64,13 @@ export function install(rt, host) {
}
const password = rt.secretValue(mailSecretName(account.address, 'PASSWORD'))
if (password === undefined) throw new Error(`The password for ${account.address} is not saved`)
- const factory = rt.config.mail?.connect ?? connectImap
- const session = await factory(account, { password })
- sessions.set(key, session)
- return session
+ return openImap(account, password)
}
+ const pending = make()
+ connecting.set(key, pending)
try {
- return await make()
- } catch (error) {
- sessions.delete(key)
- throw error
- }
+ return await pending
+ } finally { connecting.delete(key) }
}
// Gmail tokens refresh through the same OAuth rules as connectors.
@@ -281,30 +295,43 @@ export function install(rt, host) {
if (!googleReady()) throw new Error('Google sign-in is not available in this build')
return beginGoogle(address)
}
- const password = String(body.password ?? '').trim()
- if (password === '') throw new Error('Enter the authorization code or app password')
- const preset = presetOf(address)
- const provider = typeof body.provider === 'string' ? body.provider : preset?.id ?? 'other'
- const chosen = PRESETS.find(p => p.id === provider)
- const imap = chosen && provider !== 'other' ? chosen.imap : { host: String(body.imapHost ?? '').trim(), port: Number(body.imapPort) || 993, secure: body.imapTls !== false }
- const smtp = chosen && provider !== 'other' ? chosen.smtp : { host: String(body.smtpHost ?? '').trim(), port: Number(body.smtpPort) || 465, secure: body.smtpTls !== false }
- if (imap.host === '' || smtp.host === '') throw new Error('Enter the IMAP and SMTP hosts')
- const id = `m${randomUUID().slice(0, 8)}`
- const account = { id, address, provider, auth: 'password', imap, smtp, createdAt: Date.now() }
- // Log in once before saving, so a bad password never lands in state.
- const factory = rt.config.mail?.connect ?? connectImap
- let session
- try {
- session = await factory(account, { password })
- } catch (error) {
- throw new Error(mailError(error))
- }
- state.connectors = { ...(state.connectors ?? {}), mail: { ...state.connectors?.mail, accounts: { ...accounts(), [id]: account } } }
- await host.secrets.writeSecret(mailSecretName(address, 'PASSWORD'), password, `Mailbox ${address}`)
- await rt.save()
- sessions.set(id, session)
- syncTools()
- return mailView()
+ if (adding.has(address)) return adding.get(address)
+ const pending = (async () => {
+ const password = String(body.password ?? '').trim()
+ if (password === '') throw new Error('Enter the authorization code or app password')
+ const preset = presetOf(address)
+ const provider = typeof body.provider === 'string' ? body.provider : preset?.id ?? 'other'
+ const chosen = PRESETS.find(p => p.id === provider)
+ const imap = chosen && provider !== 'other' ? chosen.imap : { host: String(body.imapHost ?? '').trim(), port: Number(body.imapPort) || 993, secure: body.imapTls !== false }
+ const smtp = chosen && provider !== 'other' ? chosen.smtp : { host: String(body.smtpHost ?? '').trim(), port: Number(body.smtpPort) || 465, secure: body.smtpTls !== false }
+ if (imap.host === '' || smtp.host === '') throw new Error('Enter the IMAP and SMTP hosts')
+ const existing = accountOf(address)
+ const id = existing?.id ?? `m${randomUUID().slice(0, 8)}`
+ const account = { ...existing, id, address, provider, auth: 'password', imap, smtp, createdAt: existing?.createdAt ?? Date.now() }
+ delete account.error
+ // Log in once before saving, so a bad password never lands in state.
+ const old = sessions.get(id)
+ let session
+ try {
+ session = await openImap(account, password)
+ } catch (error) {
+ throw new Error(mailError(error))
+ }
+ await old?.close?.().catch(() => {})
+ try {
+ await host.secrets.writeSecret(mailSecretName(address, 'PASSWORD'), password, `Mailbox ${address}`)
+ state.connectors = { ...(state.connectors ?? {}), mail: { ...state.connectors?.mail, accounts: { ...accounts(), [id]: account } } }
+ await rt.save()
+ } catch (error) {
+ if (sessions.get(id) === session) sessions.delete(id)
+ await session.close?.()
+ throw error
+ }
+ syncTools()
+ return mailView()
+ })()
+ adding.set(address, pending)
+ try { return await pending } finally { adding.delete(address) }
},
async 'mail-account-remove'(body) {
const account = accountOf(body.id ?? body.address)
diff --git a/ds-bot/src/host/connectors/mail/transport.js b/ds-bot/src/host/connectors/mail/transport.js
index 74a65e2e..1e374621 100644
--- a/ds-bot/src/host/connectors/mail/transport.js
+++ b/ds-bot/src/host/connectors/mail/transport.js
@@ -2,6 +2,7 @@
// behind a small interface the tools and tests share; tests inject fakes
// through rt.config.mail.connect.
import { simpleParser } from 'mailparser'
+import { Socket } from 'node:net'
import { IMAP_ID } from './presets.js'
// IMAP + SMTP (imapflow + nodemailer). One live IMAP connection per account is
@@ -17,16 +18,61 @@ export const imapOptions = (account, deps = {}) => ({
})
export async function connectImap(account, deps = {}) {
- const { ImapFlow } = await import('imapflow')
- const nodemailer = await import('nodemailer')
+ const { ImapFlow } = deps.modules ?? await import('imapflow')
+ const nodemailer = deps.modules?.nodemailer ?? await import('nodemailer')
const imap = new ImapFlow(imapOptions(account, deps))
- const smtp = nodemailer.createTransport({
+ let gone = false
+ let smtp
+ const sockets = new Set()
+ const retire = () => {
+ if (gone) return
+ gone = true
+ imap.close()
+ for (const socket of sockets) socket.destroy()
+ smtp?.close()
+ deps.onGone?.()
+ }
+ imap.on('error', retire)
+ imap.on('close', retire)
+ smtp = nodemailer.createTransport({
host: account.smtp.host, port: account.smtp.port, secure: account.smtp.secure === true,
requireTLS: account.smtp.starttls === true,
auth: { user: account.address, pass: deps.password },
+ // Nodemailer's close() does not cancel a non-pooled verify(). Own its sockets
+ // through the transport hook so an IMAP failure also cancels that login.
+ getSocket(options, done) {
+ if (gone) { done(new Error('The mail connection is closed')); return }
+ const socket = new Socket()
+ sockets.add(socket)
+ let answered = false
+ const fail = error => { if (!answered) { answered = true; done(error) } }
+ socket.once('error', fail)
+ socket.on('error', retire)
+ socket.once('close', () => {
+ sockets.delete(socket)
+ fail(new Error('The SMTP connection closed'))
+ })
+ // Preserve Nodemailer's documented default connection timeout.
+ socket.setTimeout(options.connectionTimeout ?? 120_000, () => socket.destroy(new Error('SMTP connection timeout')))
+ socket.connect({ host: options.host, port: options.port }, () => {
+ if (answered) return
+ answered = true
+ socket.setTimeout(0)
+ done(null, { connection: socket })
+ })
+ },
})
- await imap.connect()
- await smtp.verify()
+ smtp.on('error', retire)
+ try {
+ await Promise.all([
+ imap.connect().then(() => { if (gone) imap.close() }),
+ smtp.verify().then(() => { if (gone) smtp.close() }),
+ ])
+ if (gone) throw new Error('The mail connection closed during verification')
+ } catch (error) {
+ retire()
+ throw error
+ }
const folders = async () => {
const list = await imap.list()
@@ -75,7 +121,8 @@ export async function connectImap(account, deps = {}) {
return {
kind: 'imap',
- async close() { try { await imap.logout() } catch { /* dropped */ } },
+ get usable() { return !gone && imap.usable },
+ async close() { try { if (!gone) await imap.logout() } catch { /* dropped */ } finally { retire() } },
async search({ folder = 'INBOX', from, to, subject, text, since, before, unread, limit = 20 }) {
const query = {}
if (from) query.from = from
diff --git a/ds-bot/test/connectors-engine.test.js b/ds-bot/test/connectors-engine.test.js
index 6ec20281..01b1af93 100644
--- a/ds-bot/test/connectors-engine.test.js
+++ b/ds-bot/test/connectors-engine.test.js
@@ -3,6 +3,7 @@
// test process (fakes listen on 127.0.0.1 or answer in memory).
import assert from 'node:assert/strict'
import { createServer } from 'node:http'
+import { EventEmitter } from 'node:events'
import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -16,6 +17,7 @@ const oauth = await import('../src/host/connectors/oauth.js')
const { createGateway } = await import('../src/host/connectors/gateway.js')
const { validateCapabilityName } = await import('../src/host/capabilities.js')
const { mailSecretName } = await import('../src/host/connectors/mail/presets.js')
+const { connectImap } = await import('../src/host/connectors/mail/transport.js')
const addressAt = domain => `${'me'}@${domain}`
@@ -461,6 +463,117 @@ const fakeSession = () => {
}
}
+test('mail_search reconnects once after an idle drop or an unusable cached session', async (t) => {
+ const made = []
+ const factory = async (_account, { onGone }) => {
+ const session = { ...fakeSession(), usable: true, gone: () => onGone() }
+ made.push(session)
+ return session
+ }
+ const home = teamHome()
+ const ctx = fakeContext()
+ const rt = createRuntime(ctx, { home, github: false, mail: { connect: factory } })
+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true }) })
+ await rt.load()
+ await rt.connectorEndpoint('mail-account-add', { address: addressAt('qq.com'), password: 'CODE' })
+ const search = rt.teamTools.find(d => d.name === 'mail_search')
+ made[0].gone()
+ assert.equal(await search.execute({}, { agent: { id: 'm1' } }), 'No matching mail.')
+ assert.equal(made.length, 2)
+ made[0].gone() // A late close from the old connection must not drop the new one.
+ await search.execute({}, { agent: { id: 'm1' } })
+ assert.equal(made.length, 2)
+ made[1].usable = false
+ await search.execute({}, { agent: { id: 'm1' } })
+ assert.equal(made.length, 3)
+})
+
+test('an idle ImapFlow error cannot crash the engine and the next search logs in once', async (t) => {
+ const clients = []
+ class ImapFlow extends EventEmitter {
+ usable = false
+ constructor() { super(); clients.push(this) }
+ async connect() { this.usable = true }
+ close() { this.usable = false; this.emit('close') }
+ async logout() { this.close() }
+ async getMailboxLock() { return { release() {} } }
+ async search() { return [] }
+ }
+ let made = 0
+ const factory = (account, deps) => {
+ made += 1
+ return connectImap(account, { ...deps, modules: {
+ ImapFlow, nodemailer: { createTransport: () => Object.assign(new EventEmitter(), { verify: async () => {}, close() {} }) },
+ } })
+ }
+ const home = teamHome()
+ const ctx = fakeContext()
+ const rt = createRuntime(ctx, { home, github: false, mail: { connect: factory } })
+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true }) })
+ await rt.load()
+ await rt.connectorEndpoint('mail-account-add', { address: addressAt('qq.com'), password: 'CODE' })
+ assert.doesNotThrow(() => clients[0].emit('error', new Error('Socket timeout')))
+ assert.equal(await rt.teamTools.find(d => d.name === 'mail_search').execute({}, { agent: { id: 'm1' } }), 'No matching mail.')
+ assert.equal(made, 2)
+})
+
+test('mail add updates the same address and keeps its verified session for tools', async (t) => {
+ const made = []
+ const home = teamHome()
+ const ctx = fakeContext()
+ const rt = createRuntime(ctx, { home, github: false, mail: { connect: async (account, deps) => {
+ const session = fakeSession()
+ made.push({ account, password: deps.password, session })
+ return session
+ } } })
+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true }) })
+ await rt.load()
+ const first = await rt.connectorEndpoint('mail-account-add', { address: addressAt('qq.com'), password: 'FIRST' })
+ const id = first.accounts[0].id
+ const search = rt.teamTools.find(d => d.name === 'mail_search')
+ await search.execute({}, { agent: { id: 'm1' } })
+ assert.equal(made.length, 1)
+ assert.equal(made[0].session.calls.some(c => c[0] === 'close'), false)
+ const updated = await rt.connectorEndpoint('mail-account-add', {
+ address: addressAt('qq.com').toUpperCase(), password: 'SECOND', provider: 'other',
+ imapHost: 'imap.example.test', smtpHost: 'smtp.example.test',
+ })
+ assert.equal(updated.accounts.length, 1)
+ assert.equal(updated.accounts[0].id, id)
+ assert.equal(made.length, 2)
+ assert.equal(made[1].account.imap.host, 'imap.example.test')
+ assert.equal(rt.secretValue(mailSecretName(addressAt('qq.com'), 'PASSWORD')), 'SECOND')
+ assert.equal(made[0].session.calls.some(c => c[0] === 'close'), true)
+ await search.execute({}, { agent: { id: 'm1' } })
+ assert.equal(made.length, 2)
+ assert.equal(made[1].session.calls.some(c => c[0] === 'search'), true)
+})
+
+test('concurrent adds for one address share verification and its result', async (t) => {
+ const gate = Promise.withResolvers()
+ const started = Promise.withResolvers()
+ let made = 0
+ const home = teamHome()
+ const ctx = fakeContext()
+ const rt = createRuntime(ctx, { home, github: false, mail: { connect: async () => {
+ made += 1
+ started.resolve()
+ await gate.promise
+ return fakeSession()
+ } } })
+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true }) })
+ await rt.load()
+ const first = rt.connectorEndpoint('mail-account-add', { address: addressAt('qq.com'), password: 'CODE' })
+ await started.promise
+ const second = rt.connectorEndpoint('mail-account-add', { address: addressAt('qq.com').toUpperCase(), password: 'CODE' })
+ await new Promise(resolve => setImmediate(resolve))
+ gate.resolve()
+ const [a, b] = await Promise.all([first, second])
+ assert.equal(made, 1)
+ assert.deepEqual(a, b)
+ assert.equal(rt.mailView().accounts.length, 1)
+})
+
test('mail account add verifies before saving; a bad code fails cleanly; send needs the card', async (t) => {
const sessions = new Map()
const factory = async (account, deps) => {
diff --git a/ds-bot/test/connectors-mail-transport.test.js b/ds-bot/test/connectors-mail-transport.test.js
new file mode 100644
index 00000000..a1b916c7
--- /dev/null
+++ b/ds-bot/test/connectors-mail-transport.test.js
@@ -0,0 +1,135 @@
+import assert from 'node:assert/strict'
+import { EventEmitter } from 'node:events'
+import { createServer } from 'node:net'
+import { test } from 'node:test'
+import { connectImap } from '../src/host/connectors/mail/transport.js'
+
+const account = {
+ address: '[email protected]',
+ imap: { host: '127.0.0.1', port: 1 },
+ smtp: { host: '127.0.0.1', port: 1 },
+}
+
+const transport = ({ connect = async () => {}, verify = async () => {} } = {}) => {
+ let imap
+ class ImapFlow extends EventEmitter {
+ usable = false
+ closed = false
+ constructor() { super(); imap = this }
+ async connect() { await connect(); this.usable = true }
+ close() { this.closed = true; this.usable = false; this.emit('close') }
+ async logout() { this.close() }
+ }
+ const smtp = new EventEmitter()
+ smtp.verify = verify
+ smtp.closed = false
+ smtp.close = () => { smtp.closed = true }
+ return { modules: { ImapFlow, nodemailer: { createTransport: () => smtp } }, get imap() { return imap }, smtp }
+}
+
+test('idle IMAP errors and close retire the session without escaping', async () => {
+ const fake = transport()
+ let gone = 0
+ const session = await connectImap(account, { modules: fake.modules, onGone: () => { gone += 1 } })
+ assert.equal(session.usable, true)
+ assert.doesNotThrow(() => fake.imap.emit('error', new Error('Socket timeout')))
+ assert.equal(session.usable, false)
+ fake.imap.emit('close')
+ assert.equal(gone, 1)
+ assert.equal(fake.smtp.closed, true)
+})
+
+test('SMTP errors cannot escape and close both transports', async () => {
+ const fake = transport()
+ const session = await connectImap(account, { modules: fake.modules })
+ assert.doesNotThrow(() => fake.smtp.emit('error', new Error('SMTP dropped')))
+ assert.equal(session.usable, false)
+ assert.equal(fake.imap.closed, true)
+ assert.equal(fake.smtp.closed, true)
+})
+
+test('IMAP connect and SMTP verify start together', async () => {
+ const imapGate = Promise.withResolvers()
+ const smtpStarted = Promise.withResolvers()
+ const fake = transport({ connect: () => imapGate.promise, verify: async () => { smtpStarted.resolve() } })
+ const pending = connectImap(account, { modules: fake.modules })
+ try {
+ await Promise.race([smtpStarted.promise, new Promise((_, reject) => setTimeout(() => reject(new Error('SMTP waited for IMAP')), 100))])
+ } finally { imapGate.resolve(); await (await pending).close() }
+})
+
+for (const side of ['IMAP', 'SMTP']) {
+ test(`${side} verification failure closes both transports, including a late connection`, async () => {
+ const gate = Promise.withResolvers()
+ const fail = async () => { throw new Error(`${side} refused`) }
+ const fake = transport({
+ connect: side === 'IMAP' ? fail : () => gate.promise,
+ verify: side === 'SMTP' ? fail : () => gate.promise,
+ })
+ const pending = connectImap(account, { modules: fake.modules })
+ try {
+ await assert.rejects(Promise.race([
+ pending,
+ new Promise((_, reject) => setTimeout(() => reject(new Error('verification did not reject')), 100)),
+ ]), new RegExp(`${side} refused`))
+ } finally { gate.resolve(); await pending.catch(() => {}) }
+ await new Promise(resolve => setImmediate(resolve))
+ assert.equal(fake.imap.closed, true)
+ assert.equal(fake.imap.usable, false)
+ assert.equal(fake.smtp.closed, true)
+ })
+}
+
+test('an IMAP failure cancels a real nodemailer verification socket', async (t) => {
+ const accepted = Promise.withResolvers()
+ const closed = Promise.withResolvers()
+ const server = createServer(socket => {
+ socket.on('error', () => {})
+ socket.on('close', () => closed.resolve())
+ accepted.resolve()
+ // Deliberately never send an SMTP greeting.
+ })
+ await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))
+ t.after(() => server.close())
+ const fake = transport({ connect: async () => { await accepted.promise; throw new Error('IMAP refused') } })
+ const nodemailer = await import('nodemailer')
+ await assert.rejects(connectImap({
+ ...account, smtp: { host: '127.0.0.1', port: server.address().port, secure: false },
+ }, { modules: { ImapFlow: fake.modules.ImapFlow, nodemailer } }), /IMAP refused/)
+ await closed.promise
+})
+
+test('a verified SMTP transport opens a fresh socket when sending', async (t) => {
+ let connections = 0
+ const server = createServer(socket => {
+ connections += 1
+ socket.on('error', () => {})
+ socket.write('220 fake SMTP\r\n')
+ let data = ''
+ let body = false
+ socket.on('data', chunk => {
+ data += chunk.toString()
+ let end
+ while ((end = data.indexOf('\r\n')) !== -1) {
+ const line = data.slice(0, end)
+ data = data.slice(end + 2)
+ if (body) {
+ if (line === '.') { body = false; socket.write('250 queued\r\n') }
+ } else if (line.startsWith('DATA')) { body = true; socket.write('354 send data\r\n') }
+ else if (line.startsWith('QUIT')) socket.end('221 bye\r\n')
+ else socket.write('250 OK\r\n')
+ }
+ })
+ })
+ await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))
+ t.after(() => server.close())
+ const fake = transport()
+ const nodemailer = await import('nodemailer')
+ const session = await connectImap({
+ ...account, smtp: { host: '127.0.0.1', port: server.address().port, secure: false },
+ }, { modules: { ImapFlow: fake.modules.ImapFlow, nodemailer } })
+ t.after(() => session.close())
+ await session.send({ to: '[email protected]', subject: 'Test', text: 'body' })
+ assert.equal(connections, 2)
+ assert.equal(session.usable, true)
+})